1
2 /* pnggccrd.c - mixed C/assembler version of utilities to read a PNG file
3 *
4 * For Intel/AMD x86 or x86-64 CPU (Pentium-MMX or later) and GNU C compiler.
5 *
6 * Last changed in libpng 1.2.19 August 18, 2007
7 * For conditions of distribution and use, see copyright notice in png.h
8 * Copyright (c) 1998 Intel Corporation
9 * Copyright (c) 1999-2002,2007 Greg Roelofs
10 * Copyright (c) 1998-2007 Glenn Randers-Pehrson
11 *
12 * Based on MSVC code contributed by Nirav Chhatrapati, Intel Corp., 1998.
13 * Interface to libpng contributed by Gilles Vollant, 1999.
14 * GNU C port by Greg Roelofs, 1999-2001.
15 *
16 * References:
17 *
18 * http://www.intel.com/drg/pentiumII/appnotes/916/916.htm
19 * http://www.intel.com/drg/pentiumII/appnotes/923/923.htm
20 * [Intel's performance analysis of the MMX vs. non-MMX code;
21 * moved/deleted as of 2006, but text and some graphs still
22 * available via WayBack Machine at archive.org]
23 *
24 * http://www.ibiblio.org/gferg/ldp/GCC-Inline-Assembly-HOWTO.html
25 * http://sam.zoy.org/blog/2007-04-13-shlib-with-non-pic-code-have-inline-assembly-and-pic-mix-well
26 * http://gcc.gnu.org/onlinedocs/gcc/Extended-Asm.html
27 * http://gcc.gnu.org/onlinedocs/gcc/Variable-Attributes.html
28 * http://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html
29 * AMD64 Architecture Programmer's Manual, volumes 1 and 5
30 * [http://www.amd.com/us-en/Processors/TechnicalResources/0,,30_182_739_7044,00.html]
31 * Intel 64 and IA-32 Software Developer's Manuals
32 * [http://developer.intel.com/products/processor/manuals/]
33 *
34 * png_read_filter_row_mmx_*() were converted in place with intel2gas 1.3.1:
35 *
36 * intel2gas -mdI pnggccrd.c.partially-msvc -o pnggccrd.c
37 *
38 * and then cleaned up by hand. See http://hermes.terminal.at/intel2gas/ .
39 *
40 * NOTE: A sufficiently recent version of GNU as (or as.exe under DOS/Windows)
41 * is required to assemble the newer asm instructions such as movq. (Version
42 * 2.5.2l.15 is definitely too old.) See ftp://ftp.gnu.org/pub/gnu/binutils/ .
43 */
44
45 /*
46 * PORTING NOTES AND CHANGELOG (mostly by Greg Roelofs)
47 * ===========================
48 *
49 * 19991006:
50 * - fixed sign error in post-MMX cleanup code (16- & 32-bit cases)
51 *
52 * 19991007:
53 * - additional optimizations (possible or definite):
54 * x [DONE] write MMX code for 64-bit case (pixel_bytes == 8) [not tested]
55 * - write MMX code for 48-bit case (pixel_bytes == 6)
56 * - figure out what's up with 24-bit case (pixel_bytes == 3):
57 * why subtract 8 from width_mmx in the pass 4/5 case?
58 * (only width_mmx case) (near line 2335)
59 * x [DONE] replace pixel_bytes within each block with the true
60 * constant value (or are compilers smart enough to do that?)
61 * - rewrite all MMX interlacing code so it's aligned with
62 * the *beginning* of the row buffer, not the end. This
63 * would not only allow one to eliminate half of the memory
64 * writes for odd passes (that is, pass == odd), it may also
65 * eliminate some unaligned-data-access exceptions (assuming
66 * there's a penalty for not aligning 64-bit accesses on
67 * 64-bit boundaries). The only catch is that the "leftover"
68 * pixel(s) at the end of the row would have to be saved,
69 * but there are enough unused MMX registers in every case,
70 * so this is not a problem. A further benefit is that the
71 * post-MMX cleanup code (C code) in at least some of the
72 * cases could be done within the assembler block.
73 * x [DONE] the "v3 v2 v1 v0 v7 v6 v5 v4" comments are confusing,
74 * inconsistent, and don't match the MMX Programmer's Reference
75 * Manual conventions anyway. They should be changed to
76 * "b7 b6 b5 b4 b3 b2 b1 b0," where b0 indicates the byte that
77 * was lowest in memory (i.e., corresponding to a left pixel)
78 * and b7 is the byte that was highest (i.e., a right pixel).
79 *
80 * 19991016:
81 * - Brennan's Guide notwithstanding, gcc under Linux does *not*
82 * want globals prefixed by underscores when referencing them--
83 * i.e., if the variable is const4, then refer to it as const4,
84 * not _const4. This seems to be a djgpp-specific requirement.
85 * Also, such variables apparently *must* be declared outside
86 * of functions; neither static nor automatic variables work if
87 * defined within the scope of a single function, but both
88 * static and truly global (multi-module) variables work fine.
89 *
90 * 19991017:
91 * - replaced pixel_bytes in each png_memcpy() call with constant value for
92 * inlining (png_do_read_interlace() "non-MMX/modified C code" block)
93 *
94 * 19991023:
95 * - fixed png_combine_row() non-MMX replication bug (odd passes only?)
96 * - switched from string-concatenation-with-macros to cleaner method of
97 * renaming global variables for djgpp--i.e., always use prefixes in
98 * inlined assembler code (== strings) and conditionally rename the
99 * variables, not the other way around. Hence _const4, _mask8_0, etc.
100 *
101 * 19991024:
102 * - fixed mmxsupport()/png_do_read_interlace() first-row bug
103 * This one was severely weird: even though mmxsupport() doesn't touch
104 * ebx (where "row" pointer was stored), it nevertheless managed to zero
105 * the register (even in static/non-fPIC code--see below), which in turn
106 * caused png_do_read_interlace() to return prematurely on the first row of
107 * interlaced images (i.e., without expanding the interlaced pixels).
108 * Inspection of the generated assembly code didn't turn up any clues,
109 * although it did point at a minor optimization (i.e., get rid of
110 * mmx_supported_local variable and just use eax). Possibly the CPUID
111 * instruction is more destructive than it looks? (Not yet checked.)
112 * - "info gcc" was next to useless, so compared fPIC and non-fPIC assembly
113 * listings... Apparently register spillage has to do with ebx, since
114 * it's used to index the global offset table. Commenting it out of the
115 * input-reg lists in png_combine_row() eliminated compiler barfage, so
116 * ifdef'd with __PIC__ macro: if defined, use a global for unmask
117 *
118 * 19991107:
119 * - verified CPUID clobberage: 12-char string constant ("GenuineIntel",
120 * "AuthenticAMD", etc.) placed in ebx:ecx:edx. Still need to polish.
121 *
122 * 19991120:
123 * - made "diff" variable (now "_dif") global to simplify conversion of
124 * filtering routines (running out of regs, sigh). "diff" is still used
125 * in interlacing routines, however.
126 * - fixed up both versions of mmxsupport() (ORIG_THAT_USED_TO_CLOBBER_EBX
127 * macro determines which is used); original not yet tested.
128 *
129 * 20000213:
130 * - when compiling with gcc, be sure to use -fomit-frame-pointer
131 *
132 * 20000319:
133 * - fixed a register-name typo in png_do_read_interlace(), default (MMX) case,
134 * pass == 4 or 5, that caused visible corruption of interlaced images
135 *
136 * 20000623:
137 * - Various problems were reported with gcc 2.95.2 in the Cygwin environment,
138 * many of the form "forbidden register 0 (ax) was spilled for class AREG."
139 * This is explained at http://gcc.gnu.org/fom_serv/cache/23.html, and
140 * Chuck Wilson supplied a patch involving dummy output registers. See
141 * http://sourceforge.net/bugs/?func=detailbug&bug_id=108741&group_id=5624
142 * for the original (anonymous) SourceForge bug report.
143 *
144 * 20000706:
145 * - Chuck Wilson passed along these remaining gcc 2.95.2 errors:
146 * pnggccrd.c: In function `png_combine_row':
147 * pnggccrd.c:525: more than 10 operands in `asm'
148 * pnggccrd.c:669: more than 10 operands in `asm'
149 * pnggccrd.c:828: more than 10 operands in `asm'
150 * pnggccrd.c:994: more than 10 operands in `asm'
151 * pnggccrd.c:1177: more than 10 operands in `asm'
152 * They are all the same problem and can be worked around by using the
153 * global _unmask variable unconditionally, not just in the -fPIC case.
154 * Reportedly earlier versions of gcc also have the problem with more than
155 * 10 operands; they just don't report it. Much strangeness ensues, etc.
156 *
157 * 20000729:
158 * - enabled png_read_filter_row_mmx_up() (shortest remaining unconverted
159 * MMX routine); began converting png_read_filter_row_mmx_sub()
160 * - to finish remaining sections:
161 * - clean up indentation and comments
162 * - preload local variables
163 * - add output and input regs (order of former determines numerical
164 * mapping of latter)
165 * - avoid all usage of ebx (including bx, bh, bl) register [20000823]
166 * - remove "$" from addressing of Shift and Mask variables [20000823]
167 *
168 * 20000731:
169 * - global union vars causing segfaults in png_read_filter_row_mmx_sub()?
170 *
171 * 20000822:
172 * - ARGH, stupid png_read_filter_row_mmx_sub() segfault only happens with
173 * shared-library (-fPIC) version! Code works just fine as part of static
174 * library. Should have tested that sooner.
175 * ebx is getting clobbered again (explicitly this time); need to save it
176 * on stack or rewrite asm code to avoid using it altogether. Blargh!
177 *
178 * 20000823:
179 * - first section was trickiest; all remaining sections have ebx -> edx now.
180 * (-fPIC works again.) Also added missing underscores to various Shift*
181 * and *Mask* globals and got rid of leading "$" signs.
182 *
183 * 20000826:
184 * - added visual separators to help navigate microscopic printed copies
185 * (http://pobox.com/~newt/code/gpr-latest.zip, mode 10); started working
186 * on png_read_filter_row_mmx_avg()
187 *
188 * 20000828:
189 * - finished png_read_filter_row_mmx_avg(): only Paeth left! (930 lines...)
190 * What the hell, did png_read_filter_row_mmx_paeth(), too. Comments not
191 * cleaned up/shortened in either routine, but functionality is complete
192 * and seems to be working fine.
193 *
194 * 20000829:
195 * - ahhh, figured out last(?) bit of gcc/gas asm-fu: if register is listed
196 * as an input reg (with dummy output variables, etc.), then it *cannot*
197 * also appear in the clobber list or gcc 2.95.2 will barf. The solution
198 * is simple enough...
199 *
200 * 20000914:
201 * - bug in png_read_filter_row_mmx_avg(): 16-bit grayscale not handled
202 * correctly (but 48-bit RGB just fine)
203 *
204 * 20000916:
205 * - fixed bug in png_read_filter_row_mmx_avg(), bpp == 2 case; three errors:
206 * - "_ShiftBpp.use = 24;" should have been "_ShiftBpp.use = 16;"
207 * - "_ShiftRem.use = 40;" should have been "_ShiftRem.use = 48;"
208 * - "psllq _ShiftRem, %%mm2" should have been "psrlq _ShiftRem, %%mm2"
209 *
210 * 20010101:
211 * - added new png_init_mmx_flags() function (here only because it needs to
212 * call mmxsupport(), which should probably become global png_mmxsupport());
213 * modified other MMX routines to run conditionally (png_ptr->asm_flags)
214 *
215 * 20010103:
216 * - renamed mmxsupport() to png_mmx_support(), with auto-set of mmx_supported,
217 * and made it public; moved png_init_mmx_flags() to png.c as internal func
218 *
219 * 20010104:
220 * - removed dependency on png_read_filter_row_c() (C code already duplicated
221 * within MMX version of png_read_filter_row()) so no longer necessary to
222 * compile it into pngrutil.o
223 *
224 * 20010310:
225 * - fixed buffer-overrun bug in png_combine_row() C code (non-MMX)
226 *
227 * 20010808:
228 * - added PNG_THREAD_UNSAFE_OK around code using global variables [GR-P]
229 *
230 * 20011124:
231 * - fixed missing save of Eflag in png_mmx_support() [Maxim Sobolev]
232 *
233 * 20020304:
234 * - eliminated incorrect use of width_mmx in pixel_bytes == 8 case
235 *
236 * 20020407:
237 * - fixed insufficient preservation of ebx register [Sami Farin]
238 *
239 * 20040724:
240 * - more tinkering with clobber list at lines 4529 and 5033 to get it to
241 * compile with gcc 3.4 [GR-P]
242 *
243 * 20040809:
244 * - added "rim" definitions for CONST4 and CONST6 [GR-P]
245 *
246 * 20060303:
247 * - added "OS2" to list of systems that don't need leading underscores [GR-P]
248 *
249 * 20060320:
250 * - made PIC-compliant [Christian Aichinger]
251 *
252 * 20070313:
253 * - finally applied Giuseppe Ghib�'s 64-bit patch of 20060803 (completely
254 * overlooked Dylan Alex Simon's similar patch of 20060414, oops...)
255 *
256 * 20070524:
257 * - fixed link failure caused by asm-only variables being optimized out
258 * (identified by Dimitri of Trolltech) with __attribute__((used)), which
259 * also gets rid of warnings => nuked ugly png_squelch_warnings() hack
260 * - dropped redundant ifdef
261 * - moved png_mmx_support() back up where originally intended (as in
262 * pngvcrd.c), using __attribute__((noinline)) in extra prototype
263 *
264 * 20070527:
265 * - revised png_combine_row() to reuse mask in lieu of external _unmask
266 * - moved 32-bit (RGBA) case to top of png_combine_row(): most common
267 * - just about ready to give up on x86-64 -fPIC mode; can't even access 16
268 * _mask*_* constants without triggering link error on shared library:
269 * /usr/bin/ld: pnggccrd.pic.o: relocation R_X86_64_32S against `a local
270 * symbol' can not be used when making a shared object; recompile with
271 * -fPIC
272 * pnggccrd.pic.o: could not read symbols: Bad value
273 * ("objdump -x pnggccrd.pic.o | grep rodata" to verify)
274 * [might be able to work around by doing within assembly code whatever
275 * -fPIC does, but given problems to date, seems like long shot...]
276 * [relevant ifdefs: __x86_64__ && __PIC__ => C code only]
277 * - changed #if 0 to #ifdef PNG_CLOBBER_MMX_REGS_SUPPORTED in case gcc ever
278 * supports MMX regs (%mm0, etc.) in clobber list (not supported by gcc
279 * 2.7.2.3, 2.91.66 (egcs 1.1.2), 3.x, or 4.1.2)
280 *
281 * 20070603:
282 * - revised png_combine_row() to use @GOTPCREL(%%rip) addressing on _c64
283 * struct of _mask*_* constants for x86-64 -fPIC; see sam.zoy.org link
284 * above for details
285 * - moved _const4 and _const6 into _c64 struct, renamed to _amask5_3_0 and
286 * _amask7_1_0, respectively
287 * - can't figure out how to use _c64._mask*_* vars within asm code, so still
288 * need single variables for non-x86-64/-fPIC half :-(
289 * - replaced various __PIC__ ifdefs with *_GOT_ebx macros
290 * - moved _LBCarryMask and _HBClearMask into _c64 struct
291 * - conditionally replaced _p*temp variables with %r11d-%r13d (via p*_TEMP
292 * and CLOBBER_r1*d macros)
293 *
294 * 20070604:
295 * - replaced all _ActiveMask and _ActiveMaskEnd with new _amask*_*_* consts
296 * (_amask naming convention: numbers of 00-bytes, ff-bytes, 00-bytes)
297 * - _ActiveMask // (10) // avg/paeth/sub; read-only; consts; movq/pand
298 * 0x0000000000ffffffLL (bpp 3, avg) _amask5_3_0
299 * 0xffffffffffffffffLL (bpp 4, 6, avg) _amask0_8_0
300 * 0x000000000000ffffLL (bpp 2, avg) _amask6_2_0
301 * 0x0000000000ffffffLL (bpp 3, paeth) _amask5_3_0
302 * 0x00000000ffffffffLL (bpp 6, paeth) _amask4_4_0
303 * 0x00000000ffffffffLL (bpp 4, paeth) _amask4_4_0
304 * 0x00000000ffffffffLL (bpp 8, paeth) _amask4_4_0
305 * 0x0000ffffff000000LL (bpp 3, sub) _amask2_3_3
306 * 0x00000000ffff0000LL (bpp 2, sub) _amask4_2_2
307 * - _ActiveMaskEnd // (1) // paeth only; read-only; const; pand
308 * 0xffff000000000000LL (bpp 3, paeth) _amask0_2_6
309 * - changed all "#if defined(__x86_64__) // later // && defined(__PIC__)"
310 * lines to "#ifdef PNG_x86_64_USE_GOTPCREL" for easier/safer testing
311 *
312 * 20070605:
313 * - merged PNG_x86_64_USE_GOTPCREL, non-PNG_x86_64_USE_GOTPCREL code via
314 * *MASK* and LOAD/RESTORE macros
315 *
316 * 20070607:
317 * - replaced all constant instances of _ShiftBpp, _ShiftRem with immediates
318 * (still have two shared cases in avg, sub routines)
319 *
320 * 20070609:
321 * - replaced remaining instances of _ShiftBpp, _ShiftRem with immediates
322 * (split sub and avg 4/6-bpp cases into separate blocks)
323 * - fixed paeth bug due to clobbered r11/r12/r13 regs
324 *
325 * 20070610:
326 * - made global "_dif" variable (avg/paeth/sub routines) local again (now
327 * "diff"--see 19991120 entry above), using register constraints
328 * - note that %ebp in clobber list doesn't actually work, at least for 32-bit
329 * version and gcc 4.1.2; must save and restore manually. (Seems to work
330 * OK for 64-bit version and gcc 3.4.3, but gcc may not be using ebp/rbp
331 * in that case.)
332 * - started replacing direct _MMXLength accesses with register constraints
333 *
334 * 20070612:
335 * - continued replacing direct _MMXLength accesses with register constraints
336 *
337 * 20070613:
338 * - finished replacing direct _MMXLength accesses with register constraints;
339 * switched to local variable (and renamed back to MMXLength)
340 *
341 * 20070614:
342 * - fixed sub bpp = 1 bug
343 * - started replacing direct _FullLength accesses with register constraints
344 *
345 * 20070615:
346 * - fixed 64-bit paeth bpp 3 crash bug (misplaced LOAD_GOT_rbp)
347 * - fixed 64-bit paeth bpp 1/2 and cleanup-block crash bugs (misplaced
348 * RESTORE_r11_r12_r13)
349 * - slightly optimized avg/paeth cleanup blocks and paeth bpp 1/2 block
350 * (save/restore ebx only if needed)
351 * - continued replacing direct _FullLength accesses with register constraints
352 *
353 * 20070616:
354 * - finished replacing direct _FullLength accesses with register constraints
355 * (*ugly* conditional clobber-separator macros for avg and paeth, sigh)
356 *
357 * 20070618:
358 * - fixed misplaced PNG_THREAD_UNSAFE_OK endif (was missing LOAD_GOT_rbp/
359 * RESTORE_rbp in 32-bit thread-safe case)
360 * - changed all "ifdef *" to "if defined(*)" [GR-P]
361 *
362 * 20070619:
363 * - rearranged most bitdepth-related case statements to put most frequent
364 * cases at top (24-bit, 32-bit, 8-bit, rest)
365 *
366 * 20070623:
367 * - cleaned up png_debug() warnings/formatting
368 * - removed PNG_MMX_CODE_SUPPORTED ifdefs and added outer __GNUC__ ifdef
369 * (module no longer used by non-x86/non-GCC builds as of libpng 1.2.19)
370 * - removed single libpng-1.2.x PNG_DEBUG dependency on 1.0.x png_struct
371 * member (row_buf_size)
372 * - rearranged pass-related if-blocks in png_do_read_interlace() to put most
373 * frequent cases (4, 5) at top [GR-P suggestion]
374 *
375 * 20070624-29:
376 * - fixed 64-bit crash bug: pointers -> rsi/rdi, not esi/edi (switched to
377 * %0/%1/%2/%3/%4 notation; eliminated size suffixes from relevant add/
378 * inc/sub/mov instructions; changed dummy vars to pointers)
379 * - png_combine_row()
380 * - png_do_read_interlace()
381 * - png_read_filter_row_mmx_avg()
382 * - png_read_filter_row_mmx_paeth()
383 * - png_read_filter_row_mmx_sub()
384 * - png_read_filter_row_mmx_up()
385 * - NOTE: this fix makes use of the fact that modifying a 32-bit reg (e.g.,
386 * %%ebx) clears the top half of its corresponding 64-bit reg (%%rbx), so
387 * it's safe to mix 32-bit operations with 64-bit base/index addressing
388 * (see new PSI/PAX/PBX/PDX/PBP/etc. "pointer-register" macros); applies
389 * also to clobber lists
390 *
391 * 20070630:
392 * - cleaned up formatting, macros, minor png_read_filter_row_mmx_sub() 8-bpp
393 * register-usage inefficiency
394 * - fixed 32-bit png_do_read_interlace() bug (was using pointer size for
395 * 64-bit dummy values)
396 *
397 * 20070703:
398 * - added check for (manual) PIC macro to fix OpenBSD crash bug
399 *
400 * 20070717:
401 * - fixed 48-bit png_combine_row() bug (was acting like 32-bit): copy 6
402 * bytes per pixel, not 4, and use stride of 6, not 4, in the second loop
403 * of interlace processing of 48-bit pixels [GR-P]
404 *
405 * 20070722:
406 * - fixed 64-bit png_uint_32 bug with MMXLength/FullLength temp vars
407 *
408 * [still broken: tops of all row-filter blocks (input/output constraints);
409 * shows up on 64-bit dynamic (-fPIC) version with -O2, especially if debug-
410 * printfs enabled, but at right edge of odd-width images even if disabled]
411 *
412 *
413 * STILL TO DO:
414 * - fix final thread-unsafe code using stack vars and pointer? (paeth top,
415 * default, bottom only: default, bottom already 5 reg constraints; could
416 * replace bpp with pointer and group bpp/patemp/pbtemp/pctemp in array)
417 * - fix ebp/no-reg-constraint inefficiency (avg/paeth/sub top)
418 * - test png_do_read_interlace() 64-bit case (pixel_bytes == 8)
419 * - write MMX code for 48-bit case (pixel_bytes == 6)
420 * - figure out what's up with 24-bit case (pixel_bytes == 3):
421 * why subtract 8 from width_mmx in the pass 4/5 case? due to
422 * odd number of bytes? (only width_mmx case) (near line 2335)
423 * - rewrite all MMX interlacing code so it's aligned with beginning
424 * of the row buffer, not the end (see 19991007 for details)
425 * - add error messages to any remaining bogus default cases
426 * - enable pixel_depth == 8 cases in png_read_filter_row()? (test speed)
427 * - try =r, etc., as reg constraints? (would gcc use 64-bit ones on x86-64?)
428 * - need full, non-graphical, CRC-based test suite... maybe autogenerate
429 * random data of various height/width/depth, compute CRCs, write (C
430 * funcs), read (asm/MMX), recompute CRCs, and compare?
431 * - write true x86-64 version using 128-bit "media instructions", %xmm0-15,
432 * and extra general-purpose registers
433 */
434
435 #if defined(__GNUC__)
436
437 #define PNG_INTERNAL
438 #include "png.h"
439
440
441 /* for some inexplicable reason, gcc 3.3.5 on OpenBSD (and elsewhere?) does
442 * *not* define __PIC__ when the -fPIC option is used, so we have to rely on
443 * makefiles and whatnot to define the PIC macro explicitly */
444 #if defined(PIC) && !defined(__PIC__) // (this can/should move to pngconf.h)
445 # define __PIC__
446 #endif
447
448 #if defined(PNG_ASSEMBLER_CODE_SUPPORTED) && defined(PNG_USE_PNGGCCRD)
449
450 /* if you want/need full thread-safety on x86-64 even when linking statically,
451 * comment out the "&& defined(__PIC__)" part here: */
452 #if defined(__x86_64__) && defined(__PIC__)
453 # define PNG_x86_64_USE_GOTPCREL // GOTPCREL => full thread-safety
454 # define PNG_CLOBBER_x86_64_REGS_SUPPORTED // works as of gcc 3.4.3 ...
455 #endif
456
457 int PNGAPI png_mmx_support(void);
458
459 #if defined(PNG_USE_LOCAL_ARRAYS)
460 static PNG_CONST int FARDATA png_pass_start[7] = {0, 4, 0, 2, 0, 1, 0};
461 static PNG_CONST int FARDATA png_pass_inc[7] = {8, 8, 4, 4, 2, 2, 1};
462 static PNG_CONST int FARDATA png_pass_width[7] = {8, 4, 4, 2, 2, 1, 1};
463 #endif
464
465 /* djgpp, Win32, Cygwin, and OS2 add their own underscores to global variables,
466 * so define them without: */
467 #if defined(__DJGPP__) || defined(WIN32) || defined(__CYGWIN__) || \
468 defined(__OS2__)
469 # define _mmx_supported mmx_supported
470 # define _mask8_0 mask8_0
471 # define _mask16_1 mask16_1
472 # define _mask16_0 mask16_0
473 # define _mask24_2 mask24_2
474 # define _mask24_1 mask24_1
475 # define _mask24_0 mask24_0
476 # define _mask32_3 mask32_3
477 # define _mask32_2 mask32_2
478 # define _mask32_1 mask32_1
479 # define _mask32_0 mask32_0
480 # define _mask48_5 mask48_5
481 # define _mask48_4 mask48_4
482 # define _mask48_3 mask48_3
483 # define _mask48_2 mask48_2
484 # define _mask48_1 mask48_1
485 # define _mask48_0 mask48_0
486 # define _amask5_3_0 amask5_3_0
487 # define _amask7_1_0 amask7_1_0
488 # define _LBCarryMask LBCarryMask
489 # define _HBClearMask HBClearMask
490 # define _amask0_8_0 amask0_8_0
491 # define _amask6_2_0 amask6_2_0
492 # define _amask4_4_0 amask4_4_0
493 # define _amask0_2_6 amask0_2_6
494 # define _amask2_3_3 amask2_3_3
495 # define _amask4_2_2 amask4_2_2
496 # if defined(PNG_THREAD_UNSAFE_OK)
497 # define _patemp patemp
498 # define _pbtemp pbtemp
499 # define _pctemp pctemp
500 # endif
501 #endif // djgpp, Win32, Cygwin, OS2
502
503
504 /* These constants are used in the inlined MMX assembly code. */
505
506 typedef unsigned long long ull;
507
508 #if defined(PNG_x86_64_USE_GOTPCREL)
509 static PNG_CONST struct {
510 //ull _mask_array[26];
511
512 // png_combine_row() constants:
513 ull _mask8_0;
514 ull _mask16_0, _mask16_1;
515 ull _mask24_0, _mask24_1, _mask24_2;
516 ull _mask32_0, _mask32_1, _mask32_2, _mask32_3;
517 ull _mask48_0, _mask48_1, _mask48_2, _mask48_3, _mask48_4, _mask48_5;
518
519 // png_do_read_interlace() constants:
520 ull _amask5_3_0, _amask7_1_0; // was _const4 and _const6, respectively
521
522 // png_read_filter_row_mmx_avg() constants (also uses _amask5_3_0):
523 ull _LBCarryMask, _HBClearMask;
524 ull _amask0_8_0, _amask6_2_0; // was ActiveMask for bpp 4/6 and 2 cases
525
526 // png_read_filter_row_mmx_paeth() constants (also uses _amask5_3_0):
527 ull _amask4_4_0, _amask0_2_6; // was ActiveMask{,End} for bpp 6/4/8 and 3
528
529 // png_read_filter_row_mmx_sub() constants:
530 ull _amask2_3_3, _amask4_2_2; // was ActiveMask for bpp 3 and 2 cases
531
532 } _c64 __attribute__((used, aligned(8))) = {
533
534 // png_combine_row() constants:
535 0x0102040810204080LL, // _mask8_0 offset 0
536
537 0x1010202040408080LL, // _mask16_0 offset 8
538 0x0101020204040808LL, // _mask16_1 offset 16
539
540 0x2020404040808080LL, // _mask24_0 offset 24
541 0x0408080810101020LL, // _mask24_1 offset 32
542 0x0101010202020404LL, // _mask24_2 offset 40
543
544 0x4040404080808080LL, // _mask32_0 offset 48
545 0x1010101020202020LL, // _mask32_1 offset 56
546 0x0404040408080808LL, // _mask32_2 offset 64
547 0x0101010102020202LL, // _mask32_3 offset 72
548
549 0x4040808080808080LL, // _mask48_0 offset 80
550 0x2020202040404040LL, // _mask48_1 offset 88
551 0x1010101010102020LL, // _mask48_2 offset 96
552 0x0404080808080808LL, // _mask48_3 offset 104
553 0x0202020204040404LL, // _mask48_4 offset 112
554 0x0101010101010202LL, // _mask48_5 offset 120
555
556 // png_do_read_interlace() constants:
557 0x0000000000FFFFFFLL, // _amask5_3_0 offset 128 (bpp 3, avg/paeth) const4
558 0x00000000000000FFLL, // _amask7_1_0 offset 136 const6
559
560 // png_read_filter_row_mmx_avg() constants:
561 0x0101010101010101LL, // _LBCarryMask offset 144
562 0x7F7F7F7F7F7F7F7FLL, // _HBClearMask offset 152
563 0xFFFFFFFFFFFFFFFFLL, // _amask0_8_0 offset 160 (bpp 4/6, avg)
564 0x000000000000FFFFLL, // _amask6_2_0 offset 168 (bpp 2, avg)
565
566 // png_read_filter_row_mmx_paeth() constants:
567 0x00000000FFFFFFFFLL, // _amask4_4_0 offset 176 (bpp 6/4/8, paeth)
568 0xFFFF000000000000LL, // _amask0_2_6 offset 184 (bpp 3, paeth) A.M.End
569
570 // png_read_filter_row_mmx_sub() constants:
571 0x0000FFFFFF000000LL, // _amask2_3_3 offset 192 (bpp 3, sub)
572 0x00000000FFFF0000LL, // _amask4_2_2 offset 200 (bpp 2, sub)
573
574 };
575
576 #define MASK8_0 "(%%rbp)"
577 #define MASK16_0 "8(%%rbp)"
578 #define MASK16_1 "16(%%rbp)"
579 #define MASK24_0 "24(%%rbp)"
580 #define MASK24_1 "32(%%rbp)"
581 #define MASK24_2 "40(%%rbp)"
582 #define MASK32_0 "48(%%rbp)"
583 #define MASK32_1 "56(%%rbp)"
584 #define MASK32_2 "64(%%rbp)"
585 #define MASK32_3 "72(%%rbp)"
586 #define MASK48_0 "80(%%rbp)"
587 #define MASK48_1 "88(%%rbp)"
588 #define MASK48_2 "96(%%rbp)"
589 #define MASK48_3 "104(%%rbp)"
590 #define MASK48_4 "112(%%rbp)"
591 #define MASK48_5 "120(%%rbp)"
592 #define AMASK5_3_0 "128(%%rbp)"
593 #define AMASK7_1_0 "136(%%rbp)"
594 #define LB_CARRY_MASK "144(%%rbp)"
595 #define HB_CLEAR_MASK "152(%%rbp)"
596 #define AMASK0_8_0 "160(%%rbp)"
597 #define AMASK6_2_0 "168(%%rbp)"
598 #define AMASK4_4_0 "176(%%rbp)"
599 #define AMASK0_2_6 "184(%%rbp)"
600 #define AMASK2_3_3 "192(%%rbp)"
601 #define AMASK4_2_2 "200(%%rbp)"
602
603 #else // !PNG_x86_64_USE_GOTPCREL
604
605 static PNG_CONST ull _mask8_0 __attribute__((used, aligned(8))) = 0x0102040810204080LL;
606
607 static PNG_CONST ull _mask16_1 __attribute__((used, aligned(8))) = 0x0101020204040808LL;
608 static PNG_CONST ull _mask16_0 __attribute__((used, aligned(8))) = 0x1010202040408080LL;
609
610 static PNG_CONST ull _mask24_2 __attribute__((used, aligned(8))) = 0x0101010202020404LL;
611 static PNG_CONST ull _mask24_1 __attribute__((used, aligned(8))) = 0x0408080810101020LL;
612 static PNG_CONST ull _mask24_0 __attribute__((used, aligned(8))) = 0x2020404040808080LL;
613
614 static PNG_CONST ull _mask32_3 __attribute__((used, aligned(8))) = 0x0101010102020202LL;
615 static PNG_CONST ull _mask32_2 __attribute__((used, aligned(8))) = 0x0404040408080808LL;
616 static PNG_CONST ull _mask32_1 __attribute__((used, aligned(8))) = 0x1010101020202020LL;
617 static PNG_CONST ull _mask32_0 __attribute__((used, aligned(8))) = 0x4040404080808080LL;
618
619 static PNG_CONST ull _mask48_5 __attribute__((used, aligned(8))) = 0x0101010101010202LL;
620 static PNG_CONST ull _mask48_4 __attribute__((used, aligned(8))) = 0x0202020204040404LL;
621 static PNG_CONST ull _mask48_3 __attribute__((used, aligned(8))) = 0x0404080808080808LL;
622 static PNG_CONST ull _mask48_2 __attribute__((used, aligned(8))) = 0x1010101010102020LL;
623 static PNG_CONST ull _mask48_1 __attribute__((used, aligned(8))) = 0x2020202040404040LL;
624 static PNG_CONST ull _mask48_0 __attribute__((used, aligned(8))) = 0x4040808080808080LL;
625
626 // png_do_read_interlace() constants:
627 static PNG_CONST ull _amask5_3_0 __attribute__((aligned(8))) = 0x0000000000FFFFFFLL; // was _const4
628 static PNG_CONST ull _amask7_1_0 __attribute__((aligned(8))) = 0x00000000000000FFLL; // was _const6
629
630 // png_read_filter_row_mmx_avg() constants:
631 static PNG_CONST ull _LBCarryMask __attribute__((used, aligned(8))) = 0x0101010101010101LL;
632 static PNG_CONST ull _HBClearMask __attribute__((used, aligned(8))) = 0x7f7f7f7f7f7f7f7fLL;
633 static PNG_CONST ull _amask0_8_0 __attribute__((used, aligned(8))) = 0xFFFFFFFFFFFFFFFFLL;
634 static PNG_CONST ull _amask6_2_0 __attribute__((used, aligned(8))) = 0x000000000000FFFFLL;
635
636 // png_read_filter_row_mmx_paeth() constants:
637 static PNG_CONST ull _amask4_4_0 __attribute__((used, aligned(8))) = 0x00000000FFFFFFFFLL;
638 static PNG_CONST ull _amask0_2_6 __attribute__((used, aligned(8))) = 0xFFFF000000000000LL;
639
640 // png_read_filter_row_mmx_sub() constants:
641 static PNG_CONST ull _amask2_3_3 __attribute__((used, aligned(8))) = 0x0000FFFFFF000000LL;
642 static PNG_CONST ull _amask4_2_2 __attribute__((used, aligned(8))) = 0x00000000FFFF0000LL;
643
644 #define MASK8_0 "_mask8_0"
645 #define MASK16_0 "_mask16_0"
646 #define MASK16_1 "_mask16_1"
647 #define MASK24_0 "_mask24_0"
648 #define MASK24_1 "_mask24_1"
649 #define MASK24_2 "_mask24_2"
650 #define MASK32_0 "_mask32_0"
651 #define MASK32_1 "_mask32_1"
652 #define MASK32_2 "_mask32_2"
653 #define MASK32_3 "_mask32_3"
654 #define MASK48_0 "_mask48_0"
655 #define MASK48_1 "_mask48_1"
656 #define MASK48_2 "_mask48_2"
657 #define MASK48_3 "_mask48_3"
658 #define MASK48_4 "_mask48_4"
659 #define MASK48_5 "_mask48_5"
660 #define AMASK5_3_0 "_amask5_3_0"
661 #define AMASK7_1_0 "_amask7_1_0"
662 #define LB_CARRY_MASK "_LBCarryMask"
663 #define HB_CLEAR_MASK "_HBClearMask"
664 #define AMASK0_8_0 "_amask0_8_0"
665 #define AMASK6_2_0 "_amask6_2_0"
666 #define AMASK4_4_0 "_amask4_4_0"
667 #define AMASK0_2_6 "_amask0_2_6"
668 #define AMASK2_3_3 "_amask2_3_3"
669 #define AMASK4_2_2 "_amask4_2_2"
670
671 #endif // ?PNG_x86_64_USE_GOTPCREL
672
673
674 #if defined(PNG_HAVE_MMX_READ_FILTER_ROW) || defined(PNG_HAVE_MMX_COMBINE_ROW)
675
676 // this block is specific to png_read_filter_row_mmx_paeth() except for
677 // LOAD_GOT_rbp and RESTORE_rbp, which are also used in png_combine_row()
678 #if defined(PNG_x86_64_USE_GOTPCREL)
679 # define pa_TEMP "%%r11d"
680 # define pb_TEMP "%%r12d"
681 # define pc_TEMP "%%r13d"
682 # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ...
683 # define SAVE_r11_r12_r13
684 # define RESTORE_r11_r12_r13
685 # define _CLOBBER_r11_r12_r13 ,"%r11", "%r12", "%r13"
686 # define CLOBBER_r11_r12_r13 "%r11", "%r12", "%r13"
687 # else // !PNG_CLOBBER_x86_64_REGS_SUPPORTED
688 # define SAVE_r11_r12_r13 "pushq %%r11 \n\t" \
689 "pushq %%r12 \n\t" \
690 "pushq %%r13 \n\t" // "normally 0-extended"
691 # define RESTORE_r11_r12_r13 "popq %%r13 \n\t" \
692 "popq %%r12 \n\t" \
693 "popq %%r11 \n\t"
694 # define _CLOBBER_r11_r12_r13
695 # define CLOBBER_r11_r12_r13
696 # endif
697 # define LOAD_GOT_rbp "pushq %%rbp \n\t" \
698 "movq _c64@GOTPCREL(%%rip), %%rbp \n\t"
699 # define RESTORE_rbp "popq %%rbp \n\t"
700 #else // 32-bit and/or non-PIC
701 # if defined(PNG_THREAD_UNSAFE_OK)
702 // These variables are used in png_read_filter_row_mmx_paeth() and would be
703 // local variables if not for gcc-inline-assembly addressing limitations
704 // (some apparently related to ELF format, others to CPU type).
705 //
706 // WARNING: Their presence defeats the thread-safety of libpng.
707 static int _patemp __attribute__((used));
708 static int _pbtemp __attribute__((used));
709 static int _pctemp __attribute__((used));
710 # define pa_TEMP "_patemp"
711 # define pb_TEMP "_pbtemp" // temp variables for
712 # define pc_TEMP "_pctemp" // Paeth routine
713 # define SAVE_r11_r12_r13
714 # define RESTORE_r11_r12_r13
715 # define _CLOBBER_r11_r12_r13 // not using regs => not clobbering
716 # define CLOBBER_r11_r12_r13
717 # endif // PNG_THREAD_UNSAFE_OK
718 # define LOAD_GOT_rbp
719 # define RESTORE_rbp
720 #endif
721
722 #if defined(__x86_64__)
723 # define SAVE_ebp
724 # define RESTORE_ebp
725 # define _CLOBBER_ebp ,"%ebp"
726 # define CLOBBER_ebp "%ebp"
727 # define SAVE_FullLength "movl %%eax, %%r15d \n\t"
728 # define RESTORE_FullLength "movl %%r15d, " // may go into eax or ecx
729 # if defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED) // works as of gcc 3.4.3 ...
730 # define SAVE_r15
731 # define RESTORE_r15
732 # define _CLOBBER_r15 ,"%r15"
733 # else
734 # define SAVE_r15 "pushq %%r15 \n\t"
735 # define RESTORE_r15 "popq %%r15 \n\t"
736 # define _CLOBBER_r15
737 # endif
738 # define PBP "%%rbp" // regs used for 64-bit
739 # define PAX "%%rax" // pointers or in
740 # define PBX "%%rbx" // combination with
741 # define PCX "%%rcx" // 64-bit pointer-regs
742 # define PDX "%%rdx" // (base/index pairs,
743 # define PSI "%%rsi" // add/sub/mov pairs)
744 # define CLEAR_BOTTOM_3_BITS "and $0xfffffffffffffff8, "
745 #else
746 # define SAVE_ebp "pushl %%ebp \n\t" // clobber list doesn't work
747 # define RESTORE_ebp "popl %%ebp \n\t" // for %ebp on 32-bit; not
748 # define _CLOBBER_ebp // clear why not
749 # define CLOBBER_ebp
750 # define SAVE_FullLength "pushl %%eax \n\t"
751 # define RESTORE_FullLength "popl " // eax (avg) or ecx (paeth)
752 # define SAVE_r15
753 # define RESTORE_r15
754 # define _CLOBBER_r15
755 # define PBP "%%ebp" // regs used for or in
756 # define PAX "%%eax" // combination with
757 # define PBX "%%ebx" // "normal," 32-bit
758 # define PCX "%%ecx" // pointers
759 # define PDX "%%edx"
760 # define PSI "%%esi"
761 # define CLEAR_BOTTOM_3_BITS "and $0xfffffff8, "
762 #endif
763
764 // CLOB_COMMA_ebx_ebp: need comma ONLY if both CLOBBER_ebp and CLOBBER_GOT_ebx
765 // have values, i.e., only if __x86_64__ AND !__PIC__
766 #if defined(__x86_64__) && !defined(__PIC__)
767 # define CLOB_COMMA_ebx_ebp , // clobbering both ebp and ebx => need comma
768 #else
769 # define CLOB_COMMA_ebx_ebp
770 #endif
771
772 // CLOB_COMMA_ebX_r1X: need comma UNLESS both CLOBBER_ebp and CLOBBER_GOT_ebx
773 // are empty OR CLOBBER_r11_r12_r13 is empty--i.e., NO comma
774 // if (!__x86_64__ AND __PIC__) OR !(PNG_x86_64_USE_GOTPCREL
775 // AND PNG_CLOBBER_x86_64_REGS_SUPPORTED) (double sigh...)
776 #if (!defined(__x86_64__) && defined(__PIC__)) || \
777 !defined(PNG_x86_64_USE_GOTPCREL) || \
778 !defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED)
779 # define CLOB_COMMA_ebX_r1X
780 #else
781 # define CLOB_COMMA_ebX_r1X , // clobbering (ebp OR ebx) AND r11_r12_r13
782 #endif
783
784 // CLOB_COLON_ebx_ebp: need colon unless CLOBBER_ebp and CLOBBER_GOT_ebx are
785 // BOTH empty--i.e., NO colon if (!__x86_64__ AND __PIC__)
786 // CLOB_COLON_ebx_ebp_r1X: if, in addition, CLOBBER_r11_r12_r13 is empty, then
787 // no colon for Paeth blocks, either--i.e., NO colon
788 // if !(PNG_x86_64_USE_GOTPCREL AND
789 // PNG_CLOBBER_x86_64_REGS_SUPPORTED)
790 #if (!defined(__x86_64__) && defined(__PIC__))
791 # define CLOB_COLON_ebx_ebp
792 # if !(defined(PNG_x86_64_USE_GOTPCREL) && \
793 defined(PNG_CLOBBER_x86_64_REGS_SUPPORTED))
794 # define CLOB_COLON_ebx_ebp_r1X
795 # else
796 # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13
797 # endif
798 #else
799 # define CLOB_COLON_ebx_ebp : // clobbering ebp OR ebx
800 # define CLOB_COLON_ebx_ebp_r1X : // clobbering ebp OR ebx OR r11_r12_r13
801 #endif
802
803 #endif // PNG_HAVE_MMX_READ_FILTER_ROW
804
805 #if defined(__PIC__) // macros to save, restore index to Global Offset Table
806 # if defined(__x86_64__)
807 # define SAVE_GOT_ebx "pushq %%rbx \n\t"
808 # define RESTORE_GOT_ebx "popq %%rbx \n\t"
809 # else
810 # define SAVE_GOT_ebx "pushl %%ebx \n\t"
811 # define RESTORE_GOT_ebx "popl %%ebx \n\t"
812 # endif
813 # define _CLOBBER_GOT_ebx // explicitly saved, restored => not clobbered
814 # define CLOBBER_GOT_ebx
815 #else
816 # define SAVE_GOT_ebx
817 # define RESTORE_GOT_ebx
818 # define _CLOBBER_GOT_ebx ,"%ebx"
819 # define CLOBBER_GOT_ebx "%ebx"
820 #endif
821
822 #if defined(PNG_HAVE_MMX_COMBINE_ROW) || defined(PNG_HAVE_MMX_READ_INTERLACE)
823 # define BPP2 2
824 # define BPP3 3 // bytes per pixel (a.k.a. pixel_bytes)
825 # define BPP4 4 // (defined only to help avoid cut-and-paste errors)
826 # define BPP6 6
827 # define BPP8 8
828 #endif
829
830
831
832 static int _mmx_supported = 2; // 0: no MMX; 1: MMX supported; 2: not tested
833
834 /*===========================================================================*/
835 /* */
836 /* P N G _ M M X _ S U P P O R T */
837 /* */
838 /*===========================================================================*/
839
840 // GRR NOTES: (1) the following code assumes 386 or better (pushfl/popfl)
841 // (2) all instructions compile with gcc 2.7.2.3 and later
842 // x (3) the function is moved down here to prevent gcc from
843 // x inlining it in multiple places and then barfing be-
844 // x cause the ".NOT_SUPPORTED" label is multiply defined
845 // [need to retest with gcc 2.7.2.3]
846
847 // GRR 20070524: This declaration apparently is compatible with but supersedes
848 // the one in png.h; in any case, the generated object file is slightly
849 // smaller. It is unnecessary with gcc 4.1.2, but gcc 2.x apparently
850 // replicated the ".NOT_SUPPORTED" label in each location the function was
851 // inlined, leading to compilation errors due to the "multiply defined"
852 // label. Old workaround was to leave the function at the end of this
853 // file; new one (still testing) is to use a gcc-specific function attribute
854 // to prevent local inlining.
855 int PNGAPI
856 png_mmx_support(void) __attribute__((noinline));
857
858 int PNGAPI
png_mmx_support(void)859 png_mmx_support(void)
860 {
861 #if defined(PNG_MMX_CODE_SUPPORTED) // superfluous, but what the heck
862 int result;
863 __asm__ __volatile__ (
864 #if defined(__x86_64__)
865 "pushq %%rbx \n\t" // rbx gets clobbered by CPUID instruction
866 "pushq %%rcx \n\t" // so does rcx...
867 "pushq %%rdx \n\t" // ...and rdx (but rcx & rdx safe on Linux)
868 "pushfq \n\t" // save Eflag to stack
869 "popq %%rax \n\t" // get Eflag from stack into rax
870 "movq %%rax, %%rcx \n\t" // make another copy of Eflag in rcx
871 "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
872 "pushq %%rax \n\t" // save modified Eflag back to stack
873 "popfq \n\t" // restore modified value to Eflag reg
874 "pushfq \n\t" // save Eflag to stack
875 "popq %%rax \n\t" // get Eflag from stack
876 "pushq %%rcx \n\t" // save original Eflag to stack
877 "popfq \n\t" // restore original Eflag
878 #else
879 "pushl %%ebx \n\t" // ebx gets clobbered by CPUID instruction
880 "pushl %%ecx \n\t" // so does ecx...
881 "pushl %%edx \n\t" // ...and edx (but ecx & edx safe on Linux)
882 "pushfl \n\t" // save Eflag to stack
883 "popl %%eax \n\t" // get Eflag from stack into eax
884 "movl %%eax, %%ecx \n\t" // make another copy of Eflag in ecx
885 "xorl $0x200000, %%eax \n\t" // toggle ID bit in Eflag (i.e., bit 21)
886 "pushl %%eax \n\t" // save modified Eflag back to stack
887 "popfl \n\t" // restore modified value to Eflag reg
888 "pushfl \n\t" // save Eflag to stack
889 "popl %%eax \n\t" // get Eflag from stack
890 "pushl %%ecx \n\t" // save original Eflag to stack
891 "popfl \n\t" // restore original Eflag
892 #endif
893 "xorl %%ecx, %%eax \n\t" // compare new Eflag with original Eflag
894 "jz 0f \n\t" // if same, CPUID instr. is not supported
895
896 "xorl %%eax, %%eax \n\t" // set eax to zero
897 // ".byte 0x0f, 0xa2 \n\t" // CPUID instruction (two-byte opcode)
898 "cpuid \n\t" // get the CPU identification info
899 "cmpl $1, %%eax \n\t" // make sure eax return non-zero value
900 "jl 0f \n\t" // if eax is zero, MMX is not supported
901
902 "xorl %%eax, %%eax \n\t" // set eax to zero and...
903 "incl %%eax \n\t" // ...increment eax to 1. This pair is
904 // faster than the instruction "mov eax, 1"
905 "cpuid \n\t" // get the CPU identification info again
906 "andl $0x800000, %%edx \n\t" // mask out all bits but MMX bit (23)
907 "cmpl $0, %%edx \n\t" // 0 = MMX not supported
908 "jz 0f \n\t" // non-zero = yes, MMX IS supported
909
910 "movl $1, %%eax \n\t" // set return value to 1
911 "jmp 1f \n\t" // DONE: have MMX support
912
913 "0: \n\t" // .NOT_SUPPORTED: target label for jump instructions
914 "movl $0, %%eax \n\t" // set return value to 0
915 "1: \n\t" // .RETURN: target label for jump instructions
916 #if defined(__x86_64__)
917 "popq %%rdx \n\t" // restore rdx
918 "popq %%rcx \n\t" // restore rcx
919 "popq %%rbx \n\t" // restore rbx
920 #else
921 "popl %%edx \n\t" // restore edx
922 "popl %%ecx \n\t" // restore ecx
923 "popl %%ebx \n\t" // restore ebx
924 #endif
925
926 // "ret \n\t" // DONE: no MMX support
927 // (fall through to standard C "ret")
928
929 : "=a" (result) // output list
930
931 : // any variables used on input (none)
932
933 // no clobber list
934 // , "%ebx", "%ecx", "%edx" // GRR: we handle these manually
935 // , "memory" // if write to a variable gcc thought was in a reg
936 // , "cc" // "condition codes" (flag bits)
937 );
938 _mmx_supported = result;
939 #else
940 _mmx_supported = 0;
941 #endif /* PNG_MMX_CODE_SUPPORTED */
942
943 return _mmx_supported;
944 }
945
946
947 /*===========================================================================*/
948 /* */
949 /* P N G _ C O M B I N E _ R O W */
950 /* */
951 /*===========================================================================*/
952
953 #if defined(PNG_HAVE_MMX_COMBINE_ROW)
954
955 /* Combines the row recently read in with the previous row.
956 This routine takes care of alpha and transparency if requested.
957 This routine also handles the two methods of progressive display
958 of interlaced images, depending on the mask value.
959 The mask value describes which pixels are to be combined with
960 the row. The pattern always repeats every 8 pixels, so just 8
961 bits are needed. A one indicates the pixel is to be combined; a
962 zero indicates the pixel is to be skipped. This is in addition
963 to any alpha or transparency value associated with the pixel.
964 If you want all pixels to be combined, pass 0xff (255) in mask. */
965
966 /* Use this routine for the x86 platform - it uses a faster MMX routine
967 if the machine supports MMX. */
968
969 void /* PRIVATE */
png_combine_row(png_structp png_ptr,png_bytep row,int mask)970 png_combine_row(png_structp png_ptr, png_bytep row, int mask)
971 {
972 int dummy_value_a; // fix 'forbidden register spilled' error
973 int dummy_value_c;
974 int dummy_value_d;
975 png_bytep dummy_value_S;
976 png_bytep dummy_value_D;
977
978 png_debug(1, "in png_combine_row (pnggccrd.c)\n");
979
980 if (_mmx_supported == 2) {
981 #if !defined(PNG_1_0_X)
982 /* this should have happened in png_init_mmx_flags() already */
983 png_warning(png_ptr, "asm_flags may not have been initialized");
984 #endif
985 png_mmx_support();
986 }
987
988 if (mask == 0xff)
989 {
990 png_debug(2,"mask == 0xff: doing single png_memcpy()\n");
991 png_memcpy(row, png_ptr->row_buf + 1,
992 (png_size_t)PNG_ROWBYTES(png_ptr->row_info.pixel_depth,png_ptr->width));
993 }
994 else /* (png_combine_row() is never called with mask == 0) */
995 {
996 switch (png_ptr->row_info.pixel_depth)
997 {
998 case 24: /* png_ptr->row_info.pixel_depth */
999 {
1000 png_bytep srcptr;
1001 png_bytep dstptr;
1002
1003 #if !defined(PNG_1_0_X)
1004 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1005 #else
1006 if (_mmx_supported)
1007 #endif
1008 {
1009 png_uint_32 len;
1010 int diff;
1011
1012 srcptr = png_ptr->row_buf + 1;
1013 dstptr = row;
1014 len = png_ptr->width & ~7; // reduce to multiple of 8
1015 diff = (int) (png_ptr->width & 7); // amount lost
1016
1017 __asm__ __volatile__ (
1018 "not %%edx \n\t" // mask => unmask
1019 "movd %%edx, %%mm7 \n\t" // load bit pattern
1020 "not %%edx \n\t" // unmask => mask for later
1021 "psubb %%mm6, %%mm6 \n\t" // zero mm6
1022 "punpcklbw %%mm7, %%mm7 \n\t"
1023 "punpcklwd %%mm7, %%mm7 \n\t"
1024 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1025
1026 LOAD_GOT_rbp
1027 "movq " MASK24_0 ", %%mm0 \n\t" // _mask24_0 -> mm0
1028 "movq " MASK24_1 ", %%mm1 \n\t" // _mask24_1 -> mm1
1029 "movq " MASK24_2 ", %%mm2 \n\t" // _mask24_2 -> mm2
1030 RESTORE_rbp
1031
1032 "pand %%mm7, %%mm0 \n\t"
1033 "pand %%mm7, %%mm1 \n\t"
1034 "pand %%mm7, %%mm2 \n\t"
1035
1036 "pcmpeqb %%mm6, %%mm0 \n\t"
1037 "pcmpeqb %%mm6, %%mm1 \n\t"
1038 "pcmpeqb %%mm6, %%mm2 \n\t"
1039
1040 // preload "movl len, %%ecx \n\t" // load length of line
1041 // preload "movl srcptr, %3 \n\t" // load source
1042 // preload "movl dstptr, %4 \n\t" // load dest
1043
1044 "cmpl $0, %%ecx \n\t"
1045 "jz mainloop24end \n\t"
1046
1047 "mainloop24: \n\t"
1048 "movq (%3), %%mm4 \n\t"
1049 "pand %%mm0, %%mm4 \n\t"
1050 "movq %%mm0, %%mm6 \n\t"
1051 "movq (%4), %%mm7 \n\t"
1052 "pandn %%mm7, %%mm6 \n\t"
1053 "por %%mm6, %%mm4 \n\t"
1054 "movq %%mm4, (%4) \n\t"
1055
1056 "movq 8(%3), %%mm5 \n\t"
1057 "pand %%mm1, %%mm5 \n\t"
1058 "movq %%mm1, %%mm7 \n\t"
1059 "movq 8(%4), %%mm6 \n\t"
1060 "pandn %%mm6, %%mm7 \n\t"
1061 "por %%mm7, %%mm5 \n\t"
1062 "movq %%mm5, 8(%4) \n\t"
1063
1064 "movq 16(%3), %%mm6 \n\t"
1065 "pand %%mm2, %%mm6 \n\t"
1066 "movq %%mm2, %%mm4 \n\t"
1067 "movq 16(%4), %%mm7 \n\t"
1068 "pandn %%mm7, %%mm4 \n\t"
1069 "por %%mm4, %%mm6 \n\t"
1070 "movq %%mm6, 16(%4) \n\t"
1071
1072 "add $24, %3 \n\t" // inc by 24 bytes processed
1073 "add $24, %4 \n\t"
1074 "subl $8, %%ecx \n\t" // dec by 8 pixels processed
1075
1076 "ja mainloop24 \n\t"
1077
1078 "mainloop24end: \n\t"
1079 // preload "movl diff, %%ecx \n\t" // (diff is in eax)
1080 "movl %%eax, %%ecx \n\t"
1081 "cmpl $0, %%ecx \n\t"
1082 "jz end24 \n\t"
1083 // preload "movl mask, %%edx \n\t"
1084 "sall $24, %%edx \n\t" // make low byte, high byte
1085
1086 "secondloop24: \n\t"
1087 "sall %%edx \n\t" // move high bit to CF
1088 "jnc skip24 \n\t" // if CF = 0
1089 "movw (%3), %%ax \n\t"
1090 "movw %%ax, (%4) \n\t"
1091 "xorl %%eax, %%eax \n\t"
1092 "movb 2(%3), %%al \n\t"
1093 "movb %%al, 2(%4) \n\t"
1094
1095 "skip24: \n\t"
1096 "add $3, %3 \n\t"
1097 "add $3, %4 \n\t"
1098 "decl %%ecx \n\t"
1099 "jnz secondloop24 \n\t"
1100
1101 "end24: \n\t"
1102 "EMMS \n\t" // DONE
1103
1104 : "=a" (dummy_value_a), // output regs (dummy)
1105 "=d" (dummy_value_d),
1106 "=c" (dummy_value_c),
1107 "=S" (dummy_value_S),
1108 "=D" (dummy_value_D)
1109
1110 : "0" (diff), // eax // input regs
1111 "1" (mask), // edx
1112 "2" (len), // ecx
1113 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1114 "3" (srcptr), // esi/rsi
1115 "4" (dstptr) // edi/rdi
1116
1117 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
1118 : "%mm0", "%mm1", "%mm2" // clobber list
1119 , "%mm4", "%mm5", "%mm6", "%mm7"
1120 #endif
1121 );
1122 }
1123 else /* not _mmx_supported - use modified C routine */
1124 {
1125 register png_uint_32 i;
1126 png_uint_32 initial_val = BPP3 * png_pass_start[png_ptr->pass];
1127 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1128 register int stride = BPP3 * png_pass_inc[png_ptr->pass];
1129 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1130 register int rep_bytes = BPP3 * png_pass_width[png_ptr->pass];
1131 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1132 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1133 int diff = (int) (png_ptr->width & 7); /* amount lost */
1134 register png_uint_32 final_val = BPP3 * len; /* GRR bugfix */
1135
1136 srcptr = png_ptr->row_buf + 1 + initial_val;
1137 dstptr = row + initial_val;
1138
1139 for (i = initial_val; i < final_val; i += stride)
1140 {
1141 png_memcpy(dstptr, srcptr, rep_bytes);
1142 srcptr += stride;
1143 dstptr += stride;
1144 }
1145 if (diff) /* number of leftover pixels: 3 for pngtest */
1146 {
1147 final_val += diff*BPP3;
1148 for (; i < final_val; i += stride)
1149 {
1150 if (rep_bytes > (int)(final_val-i))
1151 rep_bytes = (int)(final_val-i);
1152 png_memcpy(dstptr, srcptr, rep_bytes);
1153 srcptr += stride;
1154 dstptr += stride;
1155 }
1156 }
1157 } /* end of else (_mmx_supported) */
1158
1159 break;
1160 } /* end 24 bpp */
1161
1162 // formerly claimed to be most common case (combining 32-bit RGBA),
1163 // but almost certainly less common than 24-bit RGB case
1164 case 32: /* png_ptr->row_info.pixel_depth */
1165 {
1166 png_bytep srcptr;
1167 png_bytep dstptr;
1168
1169 #if !defined(PNG_1_0_X)
1170 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1171 #else
1172 if (_mmx_supported)
1173 #endif
1174 {
1175 png_uint_32 len;
1176 int diff;
1177
1178 srcptr = png_ptr->row_buf + 1;
1179 dstptr = row;
1180 len = png_ptr->width & ~7; // reduce to multiple of 8
1181 diff = (int) (png_ptr->width & 7); // amount lost
1182
1183 __asm__ __volatile__ (
1184 "not %%edx \n\t" // mask => unmask
1185 "movd %%edx, %%mm7 \n\t" // load bit pattern
1186 "not %%edx \n\t" // unmask => mask for later
1187 "psubb %%mm6, %%mm6 \n\t" // zero mm6
1188 "punpcklbw %%mm7, %%mm7 \n\t"
1189 "punpcklwd %%mm7, %%mm7 \n\t"
1190 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1191
1192 LOAD_GOT_rbp
1193 "movq " MASK32_0 ", %%mm0 \n\t" // _mask32_0
1194 "movq " MASK32_1 ", %%mm1 \n\t" // _mask32_1
1195 "movq " MASK32_2 ", %%mm2 \n\t" // _mask32_2
1196 "movq " MASK32_3 ", %%mm3 \n\t" // _mask32_3
1197 RESTORE_rbp
1198
1199 "pand %%mm7, %%mm0 \n\t"
1200 "pand %%mm7, %%mm1 \n\t"
1201 "pand %%mm7, %%mm2 \n\t"
1202 "pand %%mm7, %%mm3 \n\t"
1203
1204 "pcmpeqb %%mm6, %%mm0 \n\t"
1205 "pcmpeqb %%mm6, %%mm1 \n\t"
1206 "pcmpeqb %%mm6, %%mm2 \n\t"
1207 "pcmpeqb %%mm6, %%mm3 \n\t"
1208
1209 // preload "movl len, %%ecx \n\t" // load length of line
1210 // preload "movl srcptr, %3 \n\t" // load source
1211 // preload "movl dstptr, %4 \n\t" // load dest
1212
1213 "cmpl $0, %%ecx \n\t" // lcr
1214 "jz mainloop32end \n\t"
1215
1216 "mainloop32: \n\t"
1217 "movq (%3), %%mm4 \n\t"
1218 "pand %%mm0, %%mm4 \n\t"
1219 "movq %%mm0, %%mm6 \n\t"
1220 "movq (%4), %%mm7 \n\t"
1221 "pandn %%mm7, %%mm6 \n\t"
1222 "por %%mm6, %%mm4 \n\t"
1223 "movq %%mm4, (%4) \n\t"
1224
1225 "movq 8(%3), %%mm5 \n\t"
1226 "pand %%mm1, %%mm5 \n\t"
1227 "movq %%mm1, %%mm7 \n\t"
1228 "movq 8(%4), %%mm6 \n\t"
1229 "pandn %%mm6, %%mm7 \n\t"
1230 "por %%mm7, %%mm5 \n\t"
1231 "movq %%mm5, 8(%4) \n\t"
1232
1233 "movq 16(%3), %%mm6 \n\t"
1234 "pand %%mm2, %%mm6 \n\t"
1235 "movq %%mm2, %%mm4 \n\t"
1236 "movq 16(%4), %%mm7 \n\t"
1237 "pandn %%mm7, %%mm4 \n\t"
1238 "por %%mm4, %%mm6 \n\t"
1239 "movq %%mm6, 16(%4) \n\t"
1240
1241 "movq 24(%3), %%mm7 \n\t"
1242 "pand %%mm3, %%mm7 \n\t"
1243 "movq %%mm3, %%mm5 \n\t"
1244 "movq 24(%4), %%mm4 \n\t"
1245 "pandn %%mm4, %%mm5 \n\t"
1246 "por %%mm5, %%mm7 \n\t"
1247 "movq %%mm7, 24(%4) \n\t"
1248
1249 "add $32, %3 \n\t" // inc by 32 bytes processed
1250 "add $32, %4 \n\t"
1251 "subl $8, %%ecx \n\t" // dec by 8 pixels processed
1252 "ja mainloop32 \n\t"
1253
1254 "mainloop32end: \n\t"
1255 // preload "movl diff, %%ecx \n\t" // (diff is in eax)
1256 "movl %%eax, %%ecx \n\t"
1257 "cmpl $0, %%ecx \n\t"
1258 "jz end32 \n\t"
1259 // preload "movl mask, %%edx \n\t"
1260 "sall $24, %%edx \n\t" // low byte => high byte
1261
1262 "secondloop32: \n\t"
1263 "sall %%edx \n\t" // move high bit to CF
1264 "jnc skip32 \n\t" // if CF = 0
1265 "movl (%3), %%eax \n\t"
1266 "movl %%eax, (%4) \n\t"
1267
1268 "skip32: \n\t"
1269 "add $4, %3 \n\t"
1270 "add $4, %4 \n\t"
1271 "decl %%ecx \n\t"
1272 "jnz secondloop32 \n\t"
1273
1274 "end32: \n\t"
1275 "EMMS \n\t" // DONE
1276
1277 : "=a" (dummy_value_a), // output regs (dummy)
1278 "=d" (dummy_value_d),
1279 "=c" (dummy_value_c),
1280 "=S" (dummy_value_S),
1281 "=D" (dummy_value_D)
1282
1283 : "0" (diff), // eax // input regs
1284 "1" (mask), // edx
1285 "2" (len), // ecx
1286 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1287 "3" (srcptr), // esi/rsi
1288 "4" (dstptr) // edi/rdi
1289
1290 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
1291 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
1292 , "%mm4", "%mm5", "%mm6", "%mm7"
1293 #endif
1294 );
1295 }
1296 else /* not _mmx_supported - use modified C routine */
1297 {
1298 register png_uint_32 i;
1299 png_uint_32 initial_val = BPP4 * png_pass_start[png_ptr->pass];
1300 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1301 register int stride = BPP4 * png_pass_inc[png_ptr->pass];
1302 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1303 register int rep_bytes = BPP4 * png_pass_width[png_ptr->pass];
1304 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1305 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1306 int diff = (int) (png_ptr->width & 7); /* amount lost */
1307 register png_uint_32 final_val = BPP4 * len; /* GRR bugfix */
1308
1309 srcptr = png_ptr->row_buf + 1 + initial_val;
1310 dstptr = row + initial_val;
1311
1312 for (i = initial_val; i < final_val; i += stride)
1313 {
1314 png_memcpy(dstptr, srcptr, rep_bytes);
1315 srcptr += stride;
1316 dstptr += stride;
1317 }
1318 if (diff) /* number of leftover pixels: 3 for pngtest */
1319 {
1320 final_val += diff*BPP4;
1321 for (; i < final_val; i += stride)
1322 {
1323 if (rep_bytes > (int)(final_val-i))
1324 rep_bytes = (int)(final_val-i);
1325 png_memcpy(dstptr, srcptr, rep_bytes);
1326 srcptr += stride;
1327 dstptr += stride;
1328 }
1329 }
1330 } /* end of else (_mmx_supported) */
1331
1332 break;
1333 } /* end 32 bpp */
1334
1335 case 8: /* png_ptr->row_info.pixel_depth */
1336 {
1337 png_bytep srcptr;
1338 png_bytep dstptr;
1339
1340 #if !defined(PNG_1_0_X)
1341 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1342 #else
1343 if (_mmx_supported)
1344 #endif
1345 {
1346 png_uint_32 len;
1347 int diff;
1348
1349 srcptr = png_ptr->row_buf + 1;
1350 dstptr = row;
1351 len = png_ptr->width & ~7; // reduce to multiple of 8
1352 diff = (int) (png_ptr->width & 7); // amount lost
1353
1354 __asm__ __volatile__ (
1355 "not %%edx \n\t" // mask => unmask
1356 "movd %%edx, %%mm7 \n\t" // load bit pattern
1357 "not %%edx \n\t" // unmask => mask for later
1358 "psubb %%mm6, %%mm6 \n\t" // zero mm6
1359 "punpcklbw %%mm7, %%mm7 \n\t"
1360 "punpcklwd %%mm7, %%mm7 \n\t"
1361 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1362
1363 LOAD_GOT_rbp
1364 "movq " MASK8_0 ", %%mm0 \n\t" // _mask8_0 -> mm0
1365 RESTORE_rbp
1366
1367 "pand %%mm7, %%mm0 \n\t" // nonzero if keep byte
1368 "pcmpeqb %%mm6, %%mm0 \n\t" // zeros->1s, v versa
1369
1370 // preload "movl len, %%ecx \n\t" // load length of line
1371 // preload "movl srcptr, %3 \n\t" // load source
1372 // preload "movl dstptr, %4 \n\t" // load dest
1373
1374 "cmpl $0, %%ecx \n\t" // len == 0 ?
1375 "je mainloop8end \n\t"
1376
1377 "mainloop8: \n\t"
1378 "movq (%3), %%mm4 \n\t" // *srcptr
1379 "pand %%mm0, %%mm4 \n\t"
1380 "movq %%mm0, %%mm6 \n\t"
1381 "pandn (%4), %%mm6 \n\t" // *dstptr
1382 "por %%mm6, %%mm4 \n\t"
1383 "movq %%mm4, (%4) \n\t"
1384 "add $8, %3 \n\t" // inc by 8 bytes processed
1385 "add $8, %4 \n\t"
1386 "subl $8, %%ecx \n\t" // dec by 8 pixels processed
1387 "ja mainloop8 \n\t"
1388
1389 "mainloop8end: \n\t"
1390 // preload "movl diff, %%ecx \n\t" // (diff is in eax)
1391 "movl %%eax, %%ecx \n\t"
1392 "cmpl $0, %%ecx \n\t"
1393 "jz end8 \n\t"
1394 // preload "movl mask, %%edx \n\t"
1395 "sall $24, %%edx \n\t" // make low byte, high byte
1396
1397 "secondloop8: \n\t"
1398 "sall %%edx \n\t" // move high bit to CF
1399 "jnc skip8 \n\t" // if CF = 0
1400 "movb (%3), %%al \n\t"
1401 "movb %%al, (%4) \n\t"
1402
1403 "skip8: \n\t"
1404 "inc %3 \n\t"
1405 "inc %4 \n\t"
1406 "decl %%ecx \n\t"
1407 "jnz secondloop8 \n\t"
1408
1409 "end8: \n\t"
1410 "EMMS \n\t" // DONE
1411
1412 : "=a" (dummy_value_a), // output regs (dummy)
1413 "=d" (dummy_value_d),
1414 "=c" (dummy_value_c),
1415 "=S" (dummy_value_S),
1416 "=D" (dummy_value_D)
1417
1418 : "0" (diff), // eax // input regs
1419 "1" (mask), // edx
1420 "2" (len), // ecx
1421 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1422 "3" (srcptr), // esi/rsi
1423 "4" (dstptr) // edi/rdi
1424
1425 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
1426 : "%mm0", "%mm4", "%mm6", "%mm7" // clobber list
1427 #endif
1428 );
1429 }
1430 else /* not _mmx_supported - use modified C routine */
1431 {
1432 register png_uint_32 i;
1433 png_uint_32 initial_val = png_pass_start[png_ptr->pass];
1434 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1435 register int stride = png_pass_inc[png_ptr->pass];
1436 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1437 register int rep_bytes = png_pass_width[png_ptr->pass];
1438 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1439 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1440 int diff = (int) (png_ptr->width & 7); /* amount lost */
1441 register png_uint_32 final_val = len; /* GRR bugfix */
1442
1443 srcptr = png_ptr->row_buf + 1 + initial_val;
1444 dstptr = row + initial_val;
1445
1446 for (i = initial_val; i < final_val; i += stride)
1447 {
1448 png_memcpy(dstptr, srcptr, rep_bytes);
1449 srcptr += stride;
1450 dstptr += stride;
1451 }
1452 if (diff) /* number of leftover pixels: 3 for pngtest */
1453 {
1454 final_val += diff /* *BPP1 */ ;
1455 for (; i < final_val; i += stride)
1456 {
1457 if (rep_bytes > (int)(final_val-i))
1458 rep_bytes = (int)(final_val-i);
1459 png_memcpy(dstptr, srcptr, rep_bytes);
1460 srcptr += stride;
1461 dstptr += stride;
1462 }
1463 }
1464
1465 } /* end of else (_mmx_supported) */
1466
1467 break;
1468 } /* end 8 bpp */
1469
1470 case 1: /* png_ptr->row_info.pixel_depth */
1471 {
1472 png_bytep sp;
1473 png_bytep dp;
1474 int s_inc, s_start, s_end;
1475 int m;
1476 int shift;
1477 png_uint_32 i;
1478
1479 sp = png_ptr->row_buf + 1;
1480 dp = row;
1481 m = 0x80;
1482 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1483 if (png_ptr->transformations & PNG_PACKSWAP)
1484 {
1485 s_start = 0;
1486 s_end = 7;
1487 s_inc = 1;
1488 }
1489 else
1490 #endif
1491 {
1492 s_start = 7;
1493 s_end = 0;
1494 s_inc = -1;
1495 }
1496
1497 shift = s_start;
1498
1499 for (i = 0; i < png_ptr->width; i++)
1500 {
1501 if (m & mask)
1502 {
1503 int value;
1504
1505 value = (*sp >> shift) & 0x1;
1506 *dp &= (png_byte)((0x7f7f >> (7 - shift)) & 0xff);
1507 *dp |= (png_byte)(value << shift);
1508 }
1509
1510 if (shift == s_end)
1511 {
1512 shift = s_start;
1513 sp++;
1514 dp++;
1515 }
1516 else
1517 shift += s_inc;
1518
1519 if (m == 1)
1520 m = 0x80;
1521 else
1522 m >>= 1;
1523 }
1524 break;
1525 } /* end 1 bpp */
1526
1527 case 2: /* png_ptr->row_info.pixel_depth */
1528 {
1529 png_bytep sp;
1530 png_bytep dp;
1531 int s_start, s_end, s_inc;
1532 int m;
1533 int shift;
1534 png_uint_32 i;
1535 int value;
1536
1537 sp = png_ptr->row_buf + 1;
1538 dp = row;
1539 m = 0x80;
1540 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1541 if (png_ptr->transformations & PNG_PACKSWAP)
1542 {
1543 s_start = 0;
1544 s_end = 6;
1545 s_inc = 2;
1546 }
1547 else
1548 #endif
1549 {
1550 s_start = 6;
1551 s_end = 0;
1552 s_inc = -2;
1553 }
1554
1555 shift = s_start;
1556
1557 for (i = 0; i < png_ptr->width; i++)
1558 {
1559 if (m & mask)
1560 {
1561 value = (*sp >> shift) & 0x3;
1562 *dp &= (png_byte)((0x3f3f >> (6 - shift)) & 0xff);
1563 *dp |= (png_byte)(value << shift);
1564 }
1565
1566 if (shift == s_end)
1567 {
1568 shift = s_start;
1569 sp++;
1570 dp++;
1571 }
1572 else
1573 shift += s_inc;
1574 if (m == 1)
1575 m = 0x80;
1576 else
1577 m >>= 1;
1578 }
1579 break;
1580 } /* end 2 bpp */
1581
1582 case 4: /* png_ptr->row_info.pixel_depth */
1583 {
1584 png_bytep sp;
1585 png_bytep dp;
1586 int s_start, s_end, s_inc;
1587 int m;
1588 int shift;
1589 png_uint_32 i;
1590 int value;
1591
1592 sp = png_ptr->row_buf + 1;
1593 dp = row;
1594 m = 0x80;
1595 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
1596 if (png_ptr->transformations & PNG_PACKSWAP)
1597 {
1598 s_start = 0;
1599 s_end = 4;
1600 s_inc = 4;
1601 }
1602 else
1603 #endif
1604 {
1605 s_start = 4;
1606 s_end = 0;
1607 s_inc = -4;
1608 }
1609
1610 shift = s_start;
1611
1612 for (i = 0; i < png_ptr->width; i++)
1613 {
1614 if (m & mask)
1615 {
1616 value = (*sp >> shift) & 0xf;
1617 *dp &= (png_byte)((0xf0f >> (4 - shift)) & 0xff);
1618 *dp |= (png_byte)(value << shift);
1619 }
1620
1621 if (shift == s_end)
1622 {
1623 shift = s_start;
1624 sp++;
1625 dp++;
1626 }
1627 else
1628 shift += s_inc;
1629 if (m == 1)
1630 m = 0x80;
1631 else
1632 m >>= 1;
1633 }
1634 break;
1635 } /* end 4 bpp */
1636
1637 case 16: /* png_ptr->row_info.pixel_depth */
1638 {
1639 png_bytep srcptr;
1640 png_bytep dstptr;
1641
1642 #if !defined(PNG_1_0_X)
1643 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1644 #else
1645 if (_mmx_supported)
1646 #endif
1647 {
1648 png_uint_32 len;
1649 int diff;
1650
1651 srcptr = png_ptr->row_buf + 1;
1652 dstptr = row;
1653 len = png_ptr->width & ~7; // reduce to multiple of 8
1654 diff = (int) (png_ptr->width & 7); // amount lost
1655
1656 __asm__ __volatile__ (
1657 "not %%edx \n\t" // mask => unmask
1658 "movd %%edx, %%mm7 \n\t" // load bit pattern
1659 "not %%edx \n\t" // unmask => mask for later
1660 "psubb %%mm6, %%mm6 \n\t" // zero mm6
1661 "punpcklbw %%mm7, %%mm7 \n\t"
1662 "punpcklwd %%mm7, %%mm7 \n\t"
1663 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1664
1665 LOAD_GOT_rbp
1666 "movq " MASK16_0 ", %%mm0 \n\t" // _mask16_0 -> mm0
1667 "movq " MASK16_1 ", %%mm1 \n\t" // _mask16_1 -> mm1
1668 RESTORE_rbp
1669
1670 "pand %%mm7, %%mm0 \n\t"
1671 "pand %%mm7, %%mm1 \n\t"
1672
1673 "pcmpeqb %%mm6, %%mm0 \n\t"
1674 "pcmpeqb %%mm6, %%mm1 \n\t"
1675
1676 // preload "movl len, %%ecx \n\t" // load length of line
1677 // preload "movl srcptr, %3 \n\t" // load source
1678 // preload "movl dstptr, %4 \n\t" // load dest
1679
1680 "cmpl $0, %%ecx \n\t"
1681 "jz mainloop16end \n\t"
1682
1683 "mainloop16: \n\t"
1684 "movq (%3), %%mm4 \n\t"
1685 "pand %%mm0, %%mm4 \n\t"
1686 "movq %%mm0, %%mm6 \n\t"
1687 "movq (%4), %%mm7 \n\t"
1688 "pandn %%mm7, %%mm6 \n\t"
1689 "por %%mm6, %%mm4 \n\t"
1690 "movq %%mm4, (%4) \n\t"
1691
1692 "movq 8(%3), %%mm5 \n\t"
1693 "pand %%mm1, %%mm5 \n\t"
1694 "movq %%mm1, %%mm7 \n\t"
1695 "movq 8(%4), %%mm6 \n\t"
1696 "pandn %%mm6, %%mm7 \n\t"
1697 "por %%mm7, %%mm5 \n\t"
1698 "movq %%mm5, 8(%4) \n\t"
1699
1700 "add $16, %3 \n\t" // inc by 16 bytes processed
1701 "add $16, %4 \n\t"
1702 "subl $8, %%ecx \n\t" // dec by 8 pixels processed
1703 "ja mainloop16 \n\t"
1704
1705 "mainloop16end: \n\t"
1706 // preload "movl diff, %%ecx \n\t" // (diff is in eax)
1707 "movl %%eax, %%ecx \n\t"
1708 "cmpl $0, %%ecx \n\t"
1709 "jz end16 \n\t"
1710 // preload "movl mask, %%edx \n\t"
1711 "sall $24, %%edx \n\t" // make low byte, high byte
1712
1713 "secondloop16: \n\t"
1714 "sall %%edx \n\t" // move high bit to CF
1715 "jnc skip16 \n\t" // if CF = 0
1716 "movw (%3), %%ax \n\t"
1717 "movw %%ax, (%4) \n\t"
1718
1719 "skip16: \n\t"
1720 "add $2, %3 \n\t"
1721 "add $2, %4 \n\t"
1722 "decl %%ecx \n\t"
1723 "jnz secondloop16 \n\t"
1724
1725 "end16: \n\t"
1726 "EMMS \n\t" // DONE
1727
1728 : "=a" (dummy_value_a), // output regs (dummy)
1729 "=d" (dummy_value_d),
1730 "=c" (dummy_value_c),
1731 "=S" (dummy_value_S),
1732 "=D" (dummy_value_D)
1733
1734 : "0" (diff), // eax // input regs
1735 "1" (mask), // edx
1736 "2" (len), // ecx
1737 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1738 "3" (srcptr), // esi/rsi
1739 "4" (dstptr) // edi/rdi
1740
1741 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
1742 : "%mm0", "%mm1", "%mm4" // clobber list
1743 , "%mm5", "%mm6", "%mm7"
1744 #endif
1745 );
1746 }
1747 else /* not _mmx_supported - use modified C routine */
1748 {
1749 register png_uint_32 i;
1750 png_uint_32 initial_val = BPP2 * png_pass_start[png_ptr->pass];
1751 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1752 register int stride = BPP2 * png_pass_inc[png_ptr->pass];
1753 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1754 register int rep_bytes = BPP2 * png_pass_width[png_ptr->pass];
1755 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1756 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1757 int diff = (int) (png_ptr->width & 7); /* amount lost */
1758 register png_uint_32 final_val = BPP2 * len; /* GRR bugfix */
1759
1760 srcptr = png_ptr->row_buf + 1 + initial_val;
1761 dstptr = row + initial_val;
1762
1763 for (i = initial_val; i < final_val; i += stride)
1764 {
1765 png_memcpy(dstptr, srcptr, rep_bytes);
1766 srcptr += stride;
1767 dstptr += stride;
1768 }
1769 if (diff) /* number of leftover pixels: 3 for pngtest */
1770 {
1771 final_val += diff*BPP2;
1772 for (; i < final_val; i += stride)
1773 {
1774 if (rep_bytes > (int)(final_val-i))
1775 rep_bytes = (int)(final_val-i);
1776 png_memcpy(dstptr, srcptr, rep_bytes);
1777 srcptr += stride;
1778 dstptr += stride;
1779 }
1780 }
1781 } /* end of else (_mmx_supported) */
1782
1783 break;
1784 } /* end 16 bpp */
1785
1786 case 48: /* png_ptr->row_info.pixel_depth */
1787 {
1788 png_bytep srcptr;
1789 png_bytep dstptr;
1790
1791 #if !defined(PNG_1_0_X)
1792 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_COMBINE_ROW)
1793 #else
1794 if (_mmx_supported)
1795 #endif
1796 {
1797 png_uint_32 len;
1798 int diff;
1799
1800 srcptr = png_ptr->row_buf + 1;
1801 dstptr = row;
1802 len = png_ptr->width & ~7; // reduce to multiple of 8
1803 diff = (int) (png_ptr->width & 7); // amount lost
1804
1805 __asm__ __volatile__ (
1806 "not %%edx \n\t" // mask => unmask
1807 "movd %%edx, %%mm7 \n\t" // load bit pattern
1808 "not %%edx \n\t" // unmask => mask for later
1809 "psubb %%mm6, %%mm6 \n\t" // zero mm6
1810 "punpcklbw %%mm7, %%mm7 \n\t"
1811 "punpcklwd %%mm7, %%mm7 \n\t"
1812 "punpckldq %%mm7, %%mm7 \n\t" // fill reg with 8 masks
1813
1814 LOAD_GOT_rbp
1815 "movq " MASK48_0 ", %%mm0 \n\t" // _mask48_0 -> mm0
1816 "movq " MASK48_1 ", %%mm1 \n\t" // _mask48_1 -> mm1
1817 "movq " MASK48_2 ", %%mm2 \n\t" // _mask48_2 -> mm2
1818 "movq " MASK48_3 ", %%mm3 \n\t" // _mask48_3 -> mm3
1819 "movq " MASK48_4 ", %%mm4 \n\t" // _mask48_4 -> mm4
1820 "movq " MASK48_5 ", %%mm5 \n\t" // _mask48_5 -> mm5
1821 RESTORE_rbp
1822
1823 "pand %%mm7, %%mm0 \n\t"
1824 "pand %%mm7, %%mm1 \n\t"
1825 "pand %%mm7, %%mm2 \n\t"
1826 "pand %%mm7, %%mm3 \n\t"
1827 "pand %%mm7, %%mm4 \n\t"
1828 "pand %%mm7, %%mm5 \n\t"
1829
1830 "pcmpeqb %%mm6, %%mm0 \n\t"
1831 "pcmpeqb %%mm6, %%mm1 \n\t"
1832 "pcmpeqb %%mm6, %%mm2 \n\t"
1833 "pcmpeqb %%mm6, %%mm3 \n\t"
1834 "pcmpeqb %%mm6, %%mm4 \n\t"
1835 "pcmpeqb %%mm6, %%mm5 \n\t"
1836
1837 // preload "movl len, %%ecx \n\t" // load length of line
1838 // preload "movl srcptr, %3 \n\t" // load source
1839 // preload "movl dstptr, %4 \n\t" // load dest
1840
1841 "cmpl $0, %%ecx \n\t"
1842 "jz mainloop48end \n\t"
1843
1844 "mainloop48: \n\t"
1845 "movq (%3), %%mm7 \n\t"
1846 "pand %%mm0, %%mm7 \n\t"
1847 "movq %%mm0, %%mm6 \n\t"
1848 "pandn (%4), %%mm6 \n\t"
1849 "por %%mm6, %%mm7 \n\t"
1850 "movq %%mm7, (%4) \n\t"
1851
1852 "movq 8(%3), %%mm6 \n\t"
1853 "pand %%mm1, %%mm6 \n\t"
1854 "movq %%mm1, %%mm7 \n\t"
1855 "pandn 8(%4), %%mm7 \n\t"
1856 "por %%mm7, %%mm6 \n\t"
1857 "movq %%mm6, 8(%4) \n\t"
1858
1859 "movq 16(%3), %%mm6 \n\t"
1860 "pand %%mm2, %%mm6 \n\t"
1861 "movq %%mm2, %%mm7 \n\t"
1862 "pandn 16(%4), %%mm7 \n\t"
1863 "por %%mm7, %%mm6 \n\t"
1864 "movq %%mm6, 16(%4) \n\t"
1865
1866 "movq 24(%3), %%mm7 \n\t"
1867 "pand %%mm3, %%mm7 \n\t"
1868 "movq %%mm3, %%mm6 \n\t"
1869 "pandn 24(%4), %%mm6 \n\t"
1870 "por %%mm6, %%mm7 \n\t"
1871 "movq %%mm7, 24(%4) \n\t"
1872
1873 "movq 32(%3), %%mm6 \n\t"
1874 "pand %%mm4, %%mm6 \n\t"
1875 "movq %%mm4, %%mm7 \n\t"
1876 "pandn 32(%4), %%mm7 \n\t"
1877 "por %%mm7, %%mm6 \n\t"
1878 "movq %%mm6, 32(%4) \n\t"
1879
1880 "movq 40(%3), %%mm7 \n\t"
1881 "pand %%mm5, %%mm7 \n\t"
1882 "movq %%mm5, %%mm6 \n\t"
1883 "pandn 40(%4), %%mm6 \n\t"
1884 "por %%mm6, %%mm7 \n\t"
1885 "movq %%mm7, 40(%4) \n\t"
1886
1887 "add $48, %3 \n\t" // inc by 48 bytes processed
1888 "add $48, %4 \n\t"
1889 "subl $8, %%ecx \n\t" // dec by 8 pixels processed
1890
1891 "ja mainloop48 \n\t"
1892
1893 "mainloop48end: \n\t"
1894 // preload "movl diff, %%ecx \n\t" // (diff is in eax)
1895 "movl %%eax, %%ecx \n\t"
1896 "cmpl $0, %%ecx \n\t"
1897 "jz end48 \n\t"
1898 // preload "movl mask, %%edx \n\t"
1899 "sall $24, %%edx \n\t" // make low byte, high byte
1900
1901 "secondloop48: \n\t"
1902 "sall %%edx \n\t" // move high bit to CF
1903 "jnc skip48 \n\t" // if CF = 0
1904 "movl (%3), %%eax \n\t"
1905 "movl %%eax, (%4) \n\t"
1906 "movw 4(%3), %%ax \n\t" // GR-P bugfix 20070717
1907 "movw %%ax, 4(%4) \n\t" // GR-P bugfix 20070717
1908
1909 "skip48: \n\t"
1910 "add $6, %3 \n\t" // GR-P bugfix 20070717
1911 "add $6, %4 \n\t" // GR-P bugfix 20070717
1912 "decl %%ecx \n\t"
1913 "jnz secondloop48 \n\t"
1914
1915 "end48: \n\t"
1916 "EMMS \n\t" // DONE
1917
1918 : "=a" (dummy_value_a), // output regs (dummy)
1919 "=d" (dummy_value_d),
1920 "=c" (dummy_value_c),
1921 "=S" (dummy_value_S),
1922 "=D" (dummy_value_D)
1923
1924 : "0" (diff), // eax // input regs
1925 "1" (mask), // edx
1926 "2" (len), // ecx
1927 // was (unmask) "b" RESERVED // ebx // Global Offset Table idx
1928 "3" (srcptr), // esi/rsi
1929 "4" (dstptr) // edi/rdi
1930
1931 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
1932 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
1933 , "%mm4", "%mm5", "%mm6", "%mm7"
1934 #endif
1935 );
1936 }
1937 else /* not _mmx_supported - use modified C routine */
1938 {
1939 register png_uint_32 i;
1940 png_uint_32 initial_val = BPP6 * png_pass_start[png_ptr->pass];
1941 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1942 register int stride = BPP6 * png_pass_inc[png_ptr->pass];
1943 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1944 register int rep_bytes = BPP6 * png_pass_width[png_ptr->pass];
1945 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1946 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1947 int diff = (int) (png_ptr->width & 7); /* amount lost */
1948 register png_uint_32 final_val = BPP6 * len; /* GRR bugfix */
1949
1950 srcptr = png_ptr->row_buf + 1 + initial_val;
1951 dstptr = row + initial_val;
1952
1953 for (i = initial_val; i < final_val; i += stride)
1954 {
1955 png_memcpy(dstptr, srcptr, rep_bytes);
1956 srcptr += stride;
1957 dstptr += stride;
1958 }
1959 if (diff) /* number of leftover pixels: 3 for pngtest */
1960 {
1961 final_val += diff*BPP6;
1962 for (; i < final_val; i += stride)
1963 {
1964 if (rep_bytes > (int)(final_val-i))
1965 rep_bytes = (int)(final_val-i);
1966 png_memcpy(dstptr, srcptr, rep_bytes);
1967 srcptr += stride;
1968 dstptr += stride;
1969 }
1970 }
1971 } /* end of else (_mmx_supported) */
1972
1973 break;
1974 } /* end 48 bpp */
1975
1976 case 64: /* png_ptr->row_info.pixel_depth */
1977 {
1978 png_bytep srcptr;
1979 png_bytep dstptr;
1980 register png_uint_32 i;
1981 png_uint_32 initial_val = BPP8 * png_pass_start[png_ptr->pass];
1982 /* png.c: png_pass_start[] = {0, 4, 0, 2, 0, 1, 0}; */
1983 register int stride = BPP8 * png_pass_inc[png_ptr->pass];
1984 /* png.c: png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1}; */
1985 register int rep_bytes = BPP8 * png_pass_width[png_ptr->pass];
1986 /* png.c: png_pass_width[] = {8, 4, 4, 2, 2, 1, 1}; */
1987 png_uint_32 len = png_ptr->width &~7; /* reduce to mult. of 8 */
1988 int diff = (int) (png_ptr->width & 7); /* amount lost */
1989 register png_uint_32 final_val = BPP8 * len; /* GRR bugfix */
1990
1991 srcptr = png_ptr->row_buf + 1 + initial_val;
1992 dstptr = row + initial_val;
1993
1994 for (i = initial_val; i < final_val; i += stride)
1995 {
1996 png_memcpy(dstptr, srcptr, rep_bytes);
1997 srcptr += stride;
1998 dstptr += stride;
1999 }
2000 if (diff) /* number of leftover pixels: 3 for pngtest */
2001 {
2002 final_val += diff*BPP8;
2003 for (; i < final_val; i += stride)
2004 {
2005 if (rep_bytes > (int)(final_val-i))
2006 rep_bytes = (int)(final_val-i);
2007 png_memcpy(dstptr, srcptr, rep_bytes);
2008 srcptr += stride;
2009 dstptr += stride;
2010 }
2011 }
2012
2013 break;
2014 } /* end 64 bpp */
2015
2016 default: /* png_ptr->row_info.pixel_depth != 1,2,4,8,16,24,32,48,64 */
2017 {
2018 // ERROR: SHOULD NEVER BE REACHED
2019 #if defined(PNG_DEBUG)
2020 png_debug(1, "Internal libpng logic error (GCC "
2021 "png_combine_row() pixel_depth)\n");
2022 #endif
2023 break;
2024 }
2025 } /* end switch (png_ptr->row_info.pixel_depth) */
2026
2027 } /* end if (non-trivial mask) */
2028
2029 } /* end png_combine_row() */
2030
2031 #endif /* PNG_HAVE_MMX_COMBINE_ROW */
2032
2033
2034
2035
2036 /*===========================================================================*/
2037 /* */
2038 /* P N G _ D O _ R E A D _ I N T E R L A C E */
2039 /* */
2040 /*===========================================================================*/
2041
2042 #if defined(PNG_READ_INTERLACING_SUPPORTED)
2043 #if defined(PNG_HAVE_MMX_READ_INTERLACE)
2044
2045 /* png_do_read_interlace() is called after any 16-bit to 8-bit conversion
2046 * has taken place. [GRR: what other steps come before and/or after?]
2047 */
2048
2049 void /* PRIVATE */
png_do_read_interlace(png_structp png_ptr)2050 png_do_read_interlace(png_structp png_ptr)
2051 {
2052 png_row_infop row_info = &(png_ptr->row_info);
2053 png_bytep row = png_ptr->row_buf + 1;
2054 int pass = png_ptr->pass;
2055 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
2056 png_uint_32 transformations = png_ptr->transformations;
2057 #endif
2058
2059 png_debug(1, "in png_do_read_interlace (pnggccrd.c)\n");
2060
2061 if (_mmx_supported == 2) {
2062 #if !defined(PNG_1_0_X)
2063 /* this should have happened in png_init_mmx_flags() already */
2064 png_warning(png_ptr, "asm_flags may not have been initialized");
2065 #endif
2066 png_mmx_support();
2067 }
2068
2069 if (row != NULL && row_info != NULL)
2070 {
2071 png_uint_32 final_width;
2072
2073 final_width = row_info->width * png_pass_inc[pass];
2074
2075 switch (row_info->pixel_depth)
2076 {
2077 case 1:
2078 {
2079 png_bytep sp, dp;
2080 int sshift, dshift;
2081 int s_start, s_end, s_inc;
2082 png_byte v;
2083 png_uint_32 i;
2084 int j;
2085
2086 sp = row + (png_size_t)((row_info->width - 1) >> 3);
2087 dp = row + (png_size_t)((final_width - 1) >> 3);
2088 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
2089 if (transformations & PNG_PACKSWAP)
2090 {
2091 sshift = (int)((row_info->width + 7) & 7);
2092 dshift = (int)((final_width + 7) & 7);
2093 s_start = 7;
2094 s_end = 0;
2095 s_inc = -1;
2096 }
2097 else
2098 #endif
2099 {
2100 sshift = 7 - (int)((row_info->width + 7) & 7);
2101 dshift = 7 - (int)((final_width + 7) & 7);
2102 s_start = 0;
2103 s_end = 7;
2104 s_inc = 1;
2105 }
2106
2107 for (i = row_info->width; i; i--)
2108 {
2109 v = (png_byte)((*sp >> sshift) & 0x1);
2110 for (j = 0; j < png_pass_inc[pass]; j++)
2111 {
2112 *dp &= (png_byte)((0x7f7f >> (7 - dshift)) & 0xff);
2113 *dp |= (png_byte)(v << dshift);
2114 if (dshift == s_end)
2115 {
2116 dshift = s_start;
2117 dp--;
2118 }
2119 else
2120 dshift += s_inc;
2121 }
2122 if (sshift == s_end)
2123 {
2124 sshift = s_start;
2125 sp--;
2126 }
2127 else
2128 sshift += s_inc;
2129 }
2130 break;
2131 }
2132
2133 case 2:
2134 {
2135 png_bytep sp, dp;
2136 int sshift, dshift;
2137 int s_start, s_end, s_inc;
2138 png_uint_32 i;
2139
2140 sp = row + (png_size_t)((row_info->width - 1) >> 2);
2141 dp = row + (png_size_t)((final_width - 1) >> 2);
2142 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
2143 if (transformations & PNG_PACKSWAP)
2144 {
2145 sshift = (png_size_t)(((row_info->width + 3) & 3) << 1);
2146 dshift = (png_size_t)(((final_width + 3) & 3) << 1);
2147 s_start = 6;
2148 s_end = 0;
2149 s_inc = -2;
2150 }
2151 else
2152 #endif
2153 {
2154 sshift = (png_size_t)((3 - ((row_info->width + 3) & 3)) << 1);
2155 dshift = (png_size_t)((3 - ((final_width + 3) & 3)) << 1);
2156 s_start = 0;
2157 s_end = 6;
2158 s_inc = 2;
2159 }
2160
2161 for (i = row_info->width; i; i--)
2162 {
2163 png_byte v;
2164 int j;
2165
2166 v = (png_byte)((*sp >> sshift) & 0x3);
2167 for (j = 0; j < png_pass_inc[pass]; j++)
2168 {
2169 *dp &= (png_byte)((0x3f3f >> (6 - dshift)) & 0xff);
2170 *dp |= (png_byte)(v << dshift);
2171 if (dshift == s_end)
2172 {
2173 dshift = s_start;
2174 dp--;
2175 }
2176 else
2177 dshift += s_inc;
2178 }
2179 if (sshift == s_end)
2180 {
2181 sshift = s_start;
2182 sp--;
2183 }
2184 else
2185 sshift += s_inc;
2186 }
2187 break;
2188 }
2189
2190 case 4:
2191 {
2192 png_bytep sp, dp;
2193 int sshift, dshift;
2194 int s_start, s_end, s_inc;
2195 png_uint_32 i;
2196
2197 sp = row + (png_size_t)((row_info->width - 1) >> 1);
2198 dp = row + (png_size_t)((final_width - 1) >> 1);
2199 #if defined(PNG_READ_PACKSWAP_SUPPORTED)
2200 if (transformations & PNG_PACKSWAP)
2201 {
2202 sshift = (png_size_t)(((row_info->width + 1) & 1) << 2);
2203 dshift = (png_size_t)(((final_width + 1) & 1) << 2);
2204 s_start = 4;
2205 s_end = 0;
2206 s_inc = -4;
2207 }
2208 else
2209 #endif
2210 {
2211 sshift = (png_size_t)((1 - ((row_info->width + 1) & 1)) << 2);
2212 dshift = (png_size_t)((1 - ((final_width + 1) & 1)) << 2);
2213 s_start = 0;
2214 s_end = 4;
2215 s_inc = 4;
2216 }
2217
2218 for (i = row_info->width; i; i--)
2219 {
2220 png_byte v;
2221 int j;
2222
2223 v = (png_byte)((*sp >> sshift) & 0xf);
2224 for (j = 0; j < png_pass_inc[pass]; j++)
2225 {
2226 *dp &= (png_byte)((0xf0f >> (4 - dshift)) & 0xff);
2227 *dp |= (png_byte)(v << dshift);
2228 if (dshift == s_end)
2229 {
2230 dshift = s_start;
2231 dp--;
2232 }
2233 else
2234 dshift += s_inc;
2235 }
2236 if (sshift == s_end)
2237 {
2238 sshift = s_start;
2239 sp--;
2240 }
2241 else
2242 sshift += s_inc;
2243 }
2244 break;
2245 }
2246
2247 /*====================================================================*/
2248
2249 default: /* 8-bit or larger (this is where the routine is modified) */
2250 {
2251 png_bytep sptr, dp;
2252 png_uint_32 i;
2253 png_size_t pixel_bytes;
2254 int width = (int)row_info->width;
2255
2256 pixel_bytes = (row_info->pixel_depth >> 3);
2257
2258 /* point sptr at the last pixel in the pre-expanded row: */
2259 sptr = row + (width - 1) * pixel_bytes;
2260
2261 /* point dp at the last pixel position in the expanded row: */
2262 dp = row + (final_width - 1) * pixel_bytes;
2263
2264 /* New code by Nirav Chhatrapati - Intel Corporation */
2265
2266 #if !defined(PNG_1_0_X)
2267 if (png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_INTERLACE)
2268 #else
2269 if (_mmx_supported)
2270 #endif
2271 {
2272 int dummy_value_c; // fix 'forbidden register spilled'
2273 png_bytep dummy_value_S;
2274 png_bytep dummy_value_D;
2275 png_bytep dummy_value_a;
2276 png_bytep dummy_value_d;
2277
2278 //--------------------------------------------------------------
2279 if (pixel_bytes == BPP3)
2280 {
2281 if (((pass == 4) || (pass == 5)) && width)
2282 {
2283 int width_mmx = ((width >> 1) << 1) - 8; // GRR: huh?
2284 if (width_mmx < 0)
2285 width_mmx = 0;
2286 width -= width_mmx; // 8 or 9 pix, 24 or 27 bytes
2287 if (width_mmx)
2288 {
2289 // png_pass_inc[] = {8, 8, 4, 4, 2, 2, 1};
2290 // sptr points at last pixel in pre-expanded row
2291 // dp points at last pixel position in expanded row
2292 __asm__ __volatile__ (
2293 "sub $3, %1 \n\t"
2294 "sub $9, %2 \n\t"
2295 // (png_pass_inc[pass] + 1)*pixel_bytes
2296
2297 ".loop3_pass4: \n\t"
2298 "movq (%1), %%mm0 \n\t" // x x 5 4 3 2 1 0
2299 "movq %%mm0, %%mm1 \n\t" // x x 5 4 3 2 1 0
2300 "movq %%mm0, %%mm2 \n\t" // x x 5 4 3 2 1 0
2301 "psllq $24, %%mm0 \n\t" // 4 3 2 1 0 z z z
2302 "pand (%3), %%mm1 \n\t" // z z z z z 2 1 0
2303 "psrlq $24, %%mm2 \n\t" // z z z x x 5 4 3
2304 "por %%mm1, %%mm0 \n\t" // 4 3 2 1 0 2 1 0
2305 "movq %%mm2, %%mm3 \n\t" // z z z x x 5 4 3
2306 "psllq $8, %%mm2 \n\t" // z z x x 5 4 3 z
2307 "movq %%mm0, (%2) \n\t"
2308 "psrlq $16, %%mm3 \n\t" // z z z z z x x 5
2309 "pand (%4), %%mm3 \n\t" // z z z z z z z 5
2310 "por %%mm3, %%mm2 \n\t" // z z x x 5 4 3 5
2311 "sub $6, %1 \n\t"
2312 "movd %%mm2, 8(%2) \n\t"
2313 "sub $12, %2 \n\t"
2314 "subl $2, %%ecx \n\t"
2315 "jnz .loop3_pass4 \n\t"
2316 "EMMS \n\t" // DONE
2317
2318 : "=c" (dummy_value_c), // output regs (dummy)
2319 "=S" (dummy_value_S),
2320 "=D" (dummy_value_D),
2321 "=a" (dummy_value_a),
2322 "=d" (dummy_value_d)
2323
2324 : "0" (width_mmx), // ecx // input regs
2325 "1" (sptr), // esi/rsi
2326 "2" (dp), // edi/rdi
2327 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4 and _const6:
2328 "3" (&_c64._amask5_3_0), // (0x0000000000FFFFFFLL)
2329 "4" (&_c64._amask7_1_0) // (0x00000000000000FFLL)
2330 #else
2331 "3" (&_amask5_3_0), // eax (0x0000000000FFFFFFLL)
2332 "4" (&_amask7_1_0) // edx (0x00000000000000FFLL)
2333 #endif
2334
2335 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2336 : "%mm0", "%mm1" // clobber list
2337 , "%mm2", "%mm3"
2338 #endif
2339 );
2340 }
2341
2342 sptr -= width_mmx*BPP3;
2343 dp -= width_mmx*2*BPP3;
2344 for (i = width; i; i--)
2345 {
2346 png_byte v[8];
2347 int j;
2348
2349 png_memcpy(v, sptr, BPP3);
2350 for (j = 0; j < png_pass_inc[pass]; j++)
2351 {
2352 png_memcpy(dp, v, BPP3);
2353 dp -= BPP3;
2354 }
2355 sptr -= BPP3;
2356 }
2357 }
2358 else if (((pass == 2) || (pass == 3)) && width)
2359 {
2360 __asm__ __volatile__ (
2361 "sub $9, %2 \n\t"
2362 // (png_pass_inc[pass] - 1)*pixel_bytes
2363
2364 ".loop3_pass2: \n\t"
2365 "movd (%1), %%mm0 \n\t" // x x x x x 2 1 0
2366 "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0
2367 "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0
2368 "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z
2369 "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z
2370 "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z
2371 "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1
2372 "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z
2373 "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1
2374 "movq %%mm0, 4(%2) \n\t"
2375 "psrlq $16, %%mm0 \n\t" // z z 2 1 0 2 1 0
2376 "sub $3, %1 \n\t"
2377 "movd %%mm0, (%2) \n\t"
2378 "sub $12, %2 \n\t"
2379 "decl %%ecx \n\t"
2380 "jnz .loop3_pass2 \n\t"
2381 "EMMS \n\t" // DONE
2382
2383 : "=c" (dummy_value_c), // output regs (dummy)
2384 "=S" (dummy_value_S),
2385 "=D" (dummy_value_D),
2386 "=a" (dummy_value_a)
2387
2388 : "0" (width), // ecx // input regs
2389 "1" (sptr), // esi/rsi
2390 "2" (dp), // edi/rdi
2391 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4:
2392 "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL)
2393 #else
2394 "3" (&_amask5_3_0) // eax (0x0000000000FFFFFFLL)
2395 #endif
2396
2397 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
2398 : "%mm0", "%mm1", "%mm2" // clobber list
2399 #endif
2400 );
2401 }
2402 else if (width) // && ((pass == 0) || (pass == 1))
2403 {
2404 __asm__ __volatile__ (
2405 "sub $21, %2 \n\t"
2406 // (png_pass_inc[pass] - 1)*pixel_bytes
2407
2408 ".loop3_pass0: \n\t"
2409 "movd (%1), %%mm0 \n\t" // x x x x x 2 1 0
2410 "pand (%3), %%mm0 \n\t" // z z z z z 2 1 0
2411 "movq %%mm0, %%mm1 \n\t" // z z z z z 2 1 0
2412 "psllq $16, %%mm0 \n\t" // z z z 2 1 0 z z
2413 "movq %%mm0, %%mm2 \n\t" // z z z 2 1 0 z z
2414 "psllq $24, %%mm0 \n\t" // 2 1 0 z z z z z
2415 "psrlq $8, %%mm1 \n\t" // z z z z z z 2 1
2416 "por %%mm2, %%mm0 \n\t" // 2 1 0 2 1 0 z z
2417 "por %%mm1, %%mm0 \n\t" // 2 1 0 2 1 0 2 1
2418 "movq %%mm0, %%mm3 \n\t" // 2 1 0 2 1 0 2 1
2419 "psllq $16, %%mm0 \n\t" // 0 2 1 0 2 1 z z
2420 "movq %%mm3, %%mm4 \n\t" // 2 1 0 2 1 0 2 1
2421 "punpckhdq %%mm0, %%mm3 \n\t" // 0 2 1 0 2 1 0 2
2422 "movq %%mm4, 16(%2) \n\t"
2423 "psrlq $32, %%mm0 \n\t" // z z z z 0 2 1 0
2424 "movq %%mm3, 8(%2) \n\t"
2425 "punpckldq %%mm4, %%mm0 \n\t" // 1 0 2 1 0 2 1 0
2426 "sub $3, %1 \n\t"
2427 "movq %%mm0, (%2) \n\t"
2428 "sub $24, %2 \n\t"
2429 "decl %%ecx \n\t"
2430 "jnz .loop3_pass0 \n\t"
2431 "EMMS \n\t" // DONE
2432
2433 : "=c" (dummy_value_c), // output regs (dummy)
2434 "=S" (dummy_value_S),
2435 "=D" (dummy_value_D),
2436 "=a" (dummy_value_a)
2437
2438 : "0" (width), // ecx // input regs
2439 "1" (sptr), // esi/rsi
2440 "2" (dp), // edi/rdi
2441 #if defined(PNG_x86_64_USE_GOTPCREL) // formerly _const4:
2442 "3" (&_c64._amask5_3_0) // (0x0000000000FFFFFFLL)
2443 #else
2444 "3" (&_amask5_3_0) // eax (0x0000000000FFFFFFLL)
2445 #endif
2446
2447 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2448 : "%mm0", "%mm1", "%mm2" // clobber list
2449 , "%mm3", "%mm4"
2450 #endif
2451 );
2452 }
2453 } /* end of pixel_bytes == 3 */
2454
2455 //--------------------------------------------------------------
2456 else if (pixel_bytes == BPP4)
2457 {
2458 if (((pass == 4) || (pass == 5)) && width)
2459 {
2460 int width_mmx = ((width >> 1) << 1) ;
2461 width -= width_mmx; // 0,1 pixels => 0,4 bytes
2462 if (width_mmx)
2463 {
2464 __asm__ __volatile__ (
2465 "sub $4, %1 \n\t"
2466 "sub $12, %2 \n\t"
2467
2468 ".loop4_pass4: \n\t"
2469 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2470 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2471 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2472 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2473 "movq %%mm0, (%2) \n\t"
2474 "sub $8, %1 \n\t"
2475 "movq %%mm1, 8(%2) \n\t"
2476 "sub $16, %2 \n\t"
2477 "subl $2, %%ecx \n\t"
2478 "jnz .loop4_pass4 \n\t"
2479 "EMMS \n\t" // DONE
2480
2481 : "=c" (dummy_value_c), // output regs (dummy)
2482 "=S" (dummy_value_S),
2483 "=D" (dummy_value_D)
2484
2485 : "0" (width_mmx), // ecx // input regs
2486 "1" (sptr), // esi/rsi
2487 "2" (dp) // edi/rdi
2488
2489 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
2490 : "%mm0", "%mm1" // clobber list
2491 #endif
2492 );
2493 }
2494
2495 sptr -= (width_mmx*BPP4 - BPP4); // sign fixed
2496 dp -= (width_mmx*2*BPP4 - BPP4); // sign fixed
2497 for (i = width; i; i--)
2498 {
2499 png_byte v[8];
2500 int j;
2501 sptr -= BPP4;
2502 png_memcpy(v, sptr, BPP4);
2503 for (j = 0; j < png_pass_inc[pass]; j++)
2504 {
2505 dp -= BPP4;
2506 png_memcpy(dp, v, BPP4);
2507 }
2508 }
2509 }
2510 else if (((pass == 2) || (pass == 3)) && width)
2511 {
2512 int width_mmx = ((width >> 1) << 1);
2513 width -= width_mmx; // 0,1 pixels => 0,4 bytes
2514 if (width_mmx)
2515 {
2516 __asm__ __volatile__ (
2517 "sub $4, %1 \n\t"
2518 "sub $28, %2 \n\t"
2519
2520 ".loop4_pass2: \n\t"
2521 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2522 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2523 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2524 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2525 "movq %%mm0, (%2) \n\t"
2526 "movq %%mm0, 8(%2) \n\t"
2527 "movq %%mm1, 16(%2) \n\t"
2528 "movq %%mm1, 24(%2) \n\t"
2529 "sub $8, %1 \n\t"
2530 "sub $32, %2 \n\t"
2531 "subl $2, %%ecx \n\t"
2532 "jnz .loop4_pass2 \n\t"
2533 "EMMS \n\t" // DONE
2534
2535 : "=c" (dummy_value_c), // output regs (dummy)
2536 "=S" (dummy_value_S),
2537 "=D" (dummy_value_D)
2538
2539 : "0" (width_mmx), // ecx // input regs
2540 "1" (sptr), // esi/rsi
2541 "2" (dp) // edi/rdi
2542
2543 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
2544 : "%mm0", "%mm1" // clobber list
2545 #endif
2546 );
2547 }
2548
2549 sptr -= (width_mmx*4 - 4); // sign fixed
2550 dp -= (width_mmx*16 - 4); // sign fixed
2551 for (i = width; i; i--)
2552 {
2553 png_byte v[8];
2554 int j;
2555 sptr -= 4;
2556 png_memcpy(v, sptr, 4);
2557 for (j = 0; j < png_pass_inc[pass]; j++)
2558 {
2559 dp -= 4;
2560 png_memcpy(dp, v, 4);
2561 }
2562 }
2563 }
2564 else if (width) // && ((pass == 0) || (pass == 1))
2565 {
2566 int width_mmx = ((width >> 1) << 1);
2567 width -= width_mmx; // 0,1 pixels => 0,4 bytes
2568 if (width_mmx)
2569 {
2570 __asm__ __volatile__ (
2571 "sub $4, %1 \n\t"
2572 "sub $60, %2 \n\t"
2573
2574 ".loop4_pass0: \n\t"
2575 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2576 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2577 "punpckldq %%mm0, %%mm0 \n\t" // 3 2 1 0 3 2 1 0
2578 "punpckhdq %%mm1, %%mm1 \n\t" // 7 6 5 4 7 6 5 4
2579 "movq %%mm0, (%2) \n\t"
2580 "movq %%mm0, 8(%2) \n\t"
2581 "movq %%mm0, 16(%2) \n\t"
2582 "movq %%mm0, 24(%2) \n\t"
2583 "movq %%mm1, 32(%2) \n\t"
2584 "movq %%mm1, 40(%2) \n\t"
2585 "movq %%mm1, 48(%2) \n\t"
2586 "sub $8, %1 \n\t"
2587 "movq %%mm1, 56(%2) \n\t"
2588 "sub $64, %2 \n\t"
2589 "subl $2, %%ecx \n\t"
2590 "jnz .loop4_pass0 \n\t"
2591 "EMMS \n\t" // DONE
2592
2593 : "=c" (dummy_value_c), // output regs (dummy)
2594 "=S" (dummy_value_S),
2595 "=D" (dummy_value_D)
2596
2597 : "0" (width_mmx), // ecx // input regs
2598 "1" (sptr), // esi/rsi
2599 "2" (dp) // edi/rdi
2600
2601 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2602 : "%mm0", "%mm1" // clobber list
2603 #endif
2604 );
2605 }
2606
2607 sptr -= (width_mmx*4 - 4); // sign fixed
2608 dp -= (width_mmx*32 - 4); // sign fixed
2609 for (i = width; i; i--)
2610 {
2611 png_byte v[8];
2612 int j;
2613 sptr -= 4;
2614 png_memcpy(v, sptr, 4);
2615 for (j = 0; j < png_pass_inc[pass]; j++)
2616 {
2617 dp -= 4;
2618 png_memcpy(dp, v, 4);
2619 }
2620 }
2621 }
2622 } /* end of pixel_bytes == 4 */
2623
2624 //--------------------------------------------------------------
2625 else if (pixel_bytes == 1)
2626 {
2627 if (((pass == 4) || (pass == 5)) && width)
2628 {
2629 int width_mmx = ((width >> 3) << 3);
2630 width -= width_mmx; // 0-3 pixels => 0-3 bytes
2631 if (width_mmx)
2632 {
2633 __asm__ __volatile__ (
2634 "sub $7, %1 \n\t"
2635 "sub $15, %2 \n\t"
2636
2637 ".loop1_pass4: \n\t"
2638 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2639 "movq %%mm0, %%mm1 \n\t" // 7 6 5 4 3 2 1 0
2640 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
2641 "punpckhbw %%mm1, %%mm1 \n\t" // 7 7 6 6 5 5 4 4
2642 "movq %%mm1, 8(%2) \n\t"
2643 "sub $8, %1 \n\t"
2644 "movq %%mm0, (%2) \n\t"
2645 "sub $16, %2 \n\t"
2646 "subl $8, %%ecx \n\t"
2647 "jnz .loop1_pass4 \n\t"
2648 "EMMS \n\t" // DONE
2649
2650 : "=c" (dummy_value_c), // output regs (dummy)
2651 "=S" (dummy_value_S),
2652 "=D" (dummy_value_D)
2653
2654 : "0" (width_mmx), // ecx // input regs
2655 "1" (sptr), // esi/rsi
2656 "2" (dp) // edi/rdi
2657
2658 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2659 : "%mm0", "%mm1" // clobber list
2660 #endif
2661 );
2662 }
2663
2664 sptr -= width_mmx;
2665 dp -= width_mmx*2;
2666 for (i = width; i; i--)
2667 {
2668 int j;
2669
2670 for (j = 0; j < png_pass_inc[pass]; j++)
2671 {
2672 *dp-- = *sptr;
2673 }
2674 --sptr;
2675 }
2676 }
2677 else if (((pass == 2) || (pass == 3)) && width)
2678 {
2679 int width_mmx = ((width >> 2) << 2);
2680 width -= width_mmx; // 0-3 pixels => 0-3 bytes
2681 if (width_mmx)
2682 {
2683 __asm__ __volatile__ (
2684 "sub $3, %1 \n\t"
2685 "sub $15, %2 \n\t"
2686
2687 ".loop1_pass2: \n\t"
2688 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0
2689 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
2690 "movq %%mm0, %%mm1 \n\t" // 3 3 2 2 1 1 0 0
2691 "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0
2692 "punpckhwd %%mm1, %%mm1 \n\t" // 3 3 3 3 2 2 2 2
2693 "movq %%mm0, (%2) \n\t"
2694 "sub $4, %1 \n\t"
2695 "movq %%mm1, 8(%2) \n\t"
2696 "sub $16, %2 \n\t"
2697 "subl $4, %%ecx \n\t"
2698 "jnz .loop1_pass2 \n\t"
2699 "EMMS \n\t" // DONE
2700
2701 : "=c" (dummy_value_c), // output regs (dummy)
2702 "=S" (dummy_value_S),
2703 "=D" (dummy_value_D)
2704
2705 : "0" (width_mmx), // ecx // input regs
2706 "1" (sptr), // esi/rsi
2707 "2" (dp) // edi/rdi
2708
2709 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2710 : "%mm0", "%mm1" // clobber list
2711 #endif
2712 );
2713 }
2714
2715 sptr -= width_mmx;
2716 dp -= width_mmx*4;
2717 for (i = width; i; i--)
2718 {
2719 int j;
2720
2721 for (j = 0; j < png_pass_inc[pass]; j++)
2722 {
2723 *dp-- = *sptr;
2724 }
2725 --sptr;
2726 }
2727 }
2728 else if (width) // && ((pass == 0) || (pass == 1))
2729 {
2730 int width_mmx = ((width >> 2) << 2);
2731 width -= width_mmx; // 0-3 pixels => 0-3 bytes
2732 if (width_mmx)
2733 {
2734 __asm__ __volatile__ (
2735 "sub $3, %1 \n\t"
2736 "sub $31, %2 \n\t"
2737
2738 ".loop1_pass0: \n\t"
2739 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0
2740 "movq %%mm0, %%mm1 \n\t" // x x x x 3 2 1 0
2741 "punpcklbw %%mm0, %%mm0 \n\t" // 3 3 2 2 1 1 0 0
2742 "movq %%mm0, %%mm2 \n\t" // 3 3 2 2 1 1 0 0
2743 "punpcklwd %%mm0, %%mm0 \n\t" // 1 1 1 1 0 0 0 0
2744 "movq %%mm0, %%mm3 \n\t" // 1 1 1 1 0 0 0 0
2745 "punpckldq %%mm0, %%mm0 \n\t" // 0 0 0 0 0 0 0 0
2746 "punpckhdq %%mm3, %%mm3 \n\t" // 1 1 1 1 1 1 1 1
2747 "movq %%mm0, (%2) \n\t"
2748 "punpckhwd %%mm2, %%mm2 \n\t" // 3 3 3 3 2 2 2 2
2749 "movq %%mm3, 8(%2) \n\t"
2750 "movq %%mm2, %%mm4 \n\t" // 3 3 3 3 2 2 2 2
2751 "punpckldq %%mm2, %%mm2 \n\t" // 2 2 2 2 2 2 2 2
2752 "punpckhdq %%mm4, %%mm4 \n\t" // 3 3 3 3 3 3 3 3
2753 "movq %%mm2, 16(%2) \n\t"
2754 "sub $4, %1 \n\t"
2755 "movq %%mm4, 24(%2) \n\t"
2756 "sub $32, %2 \n\t"
2757 "subl $4, %%ecx \n\t"
2758 "jnz .loop1_pass0 \n\t"
2759 "EMMS \n\t" // DONE
2760
2761 : "=c" (dummy_value_c), // output regs (dummy)
2762 "=S" (dummy_value_S),
2763 "=D" (dummy_value_D)
2764
2765 : "0" (width_mmx), // ecx // input regs
2766 "1" (sptr), // esi/rsi
2767 "2" (dp) // edi/rdi
2768
2769 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2770 : "%mm0", "%mm1", "%mm2" // clobber list
2771 , "%mm3", "%mm4"
2772 #endif
2773 );
2774 }
2775
2776 sptr -= width_mmx;
2777 dp -= width_mmx*8;
2778 for (i = width; i; i--)
2779 {
2780 int j;
2781
2782 /* I simplified this part in version 1.0.4e
2783 * here and in several other instances where
2784 * pixel_bytes == 1 -- GR-P
2785 *
2786 * Original code:
2787 *
2788 * png_byte v[8];
2789 * png_memcpy(v, sptr, pixel_bytes);
2790 * for (j = 0; j < png_pass_inc[pass]; j++)
2791 * {
2792 * png_memcpy(dp, v, pixel_bytes);
2793 * dp -= pixel_bytes;
2794 * }
2795 * sptr -= pixel_bytes;
2796 *
2797 * Replacement code is in the next three lines:
2798 */
2799
2800 for (j = 0; j < png_pass_inc[pass]; j++)
2801 {
2802 *dp-- = *sptr;
2803 }
2804 --sptr;
2805 }
2806 }
2807 } /* end of pixel_bytes == 1 */
2808
2809 //--------------------------------------------------------------
2810 else if (pixel_bytes == BPP2)
2811 {
2812 if (((pass == 4) || (pass == 5)) && width)
2813 {
2814 int width_mmx = ((width >> 1) << 1) ;
2815 width -= width_mmx; // 0,1 pixels => 0,2 bytes
2816 if (width_mmx)
2817 {
2818 __asm__ __volatile__ (
2819 "sub $2, %1 \n\t"
2820 "sub $6, %2 \n\t"
2821
2822 ".loop2_pass4: \n\t"
2823 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0
2824 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2825 "sub $4, %1 \n\t"
2826 "movq %%mm0, (%2) \n\t"
2827 "sub $8, %2 \n\t"
2828 "subl $2, %%ecx \n\t"
2829 "jnz .loop2_pass4 \n\t"
2830 "EMMS \n\t" // DONE
2831
2832 : "=c" (dummy_value_c), // output regs (dummy)
2833 "=S" (dummy_value_S),
2834 "=D" (dummy_value_D)
2835
2836 : "0" (width_mmx), // ecx // input regs
2837 "1" (sptr), // esi/rsi
2838 "2" (dp) // edi/rdi
2839
2840 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
2841 : "%mm0" // clobber list
2842 #endif
2843 );
2844 }
2845
2846 sptr -= (width_mmx*BPP2 - BPP2); // sign fixed
2847 dp -= (width_mmx*2*BPP2 - BPP2); // sign fixed
2848 for (i = width; i; i--)
2849 {
2850 png_byte v[8];
2851 int j;
2852 sptr -= BPP2;
2853 png_memcpy(v, sptr, BPP2);
2854 for (j = 0; j < png_pass_inc[pass]; j++)
2855 {
2856 dp -= BPP2;
2857 png_memcpy(dp, v, BPP2);
2858 }
2859 }
2860 }
2861 else if (((pass == 2) || (pass == 3)) && width)
2862 {
2863 int width_mmx = ((width >> 1) << 1) ;
2864 width -= width_mmx; // 0,1 pixels => 0,2 bytes
2865 if (width_mmx)
2866 {
2867 __asm__ __volatile__ (
2868 "sub $2, %1 \n\t"
2869 "sub $14, %2 \n\t"
2870
2871 ".loop2_pass2: \n\t"
2872 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0
2873 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2874 "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0
2875 "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0
2876 "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2
2877 "movq %%mm0, (%2) \n\t"
2878 "sub $4, %1 \n\t"
2879 "movq %%mm1, 8(%2) \n\t"
2880 "sub $16, %2 \n\t"
2881 "subl $2, %%ecx \n\t"
2882 "jnz .loop2_pass2 \n\t"
2883 "EMMS \n\t" // DONE
2884
2885 : "=c" (dummy_value_c), // output regs (dummy)
2886 "=S" (dummy_value_S),
2887 "=D" (dummy_value_D)
2888
2889 : "0" (width_mmx), // ecx // input regs
2890 "1" (sptr), // esi/rsi
2891 "2" (dp) // edi/rdi
2892
2893 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
2894 : "%mm0", "%mm1" // clobber list
2895 #endif
2896 );
2897 }
2898
2899 sptr -= (width_mmx*2 - 2); // sign fixed
2900 dp -= (width_mmx*8 - 2); // sign fixed
2901 for (i = width; i; i--)
2902 {
2903 png_byte v[8];
2904 int j;
2905 sptr -= 2;
2906 png_memcpy(v, sptr, 2);
2907 for (j = 0; j < png_pass_inc[pass]; j++)
2908 {
2909 dp -= 2;
2910 png_memcpy(dp, v, 2);
2911 }
2912 }
2913 }
2914 else if (width) // && ((pass == 0) || (pass == 1))
2915 {
2916 int width_mmx = ((width >> 1) << 1);
2917 width -= width_mmx; // 0,1 pixels => 0,2 bytes
2918 if (width_mmx)
2919 {
2920 __asm__ __volatile__ (
2921 "sub $2, %1 \n\t"
2922 "sub $30, %2 \n\t"
2923
2924 ".loop2_pass0: \n\t"
2925 "movd (%1), %%mm0 \n\t" // x x x x 3 2 1 0
2926 "punpcklwd %%mm0, %%mm0 \n\t" // 3 2 3 2 1 0 1 0
2927 "movq %%mm0, %%mm1 \n\t" // 3 2 3 2 1 0 1 0
2928 "punpckldq %%mm0, %%mm0 \n\t" // 1 0 1 0 1 0 1 0
2929 "punpckhdq %%mm1, %%mm1 \n\t" // 3 2 3 2 3 2 3 2
2930 "movq %%mm0, (%2) \n\t"
2931 "movq %%mm0, 8(%2) \n\t"
2932 "movq %%mm1, 16(%2) \n\t"
2933 "sub $4, %1 \n\t"
2934 "movq %%mm1, 24(%2) \n\t"
2935 "sub $32, %2 \n\t"
2936 "subl $2, %%ecx \n\t"
2937 "jnz .loop2_pass0 \n\t"
2938 "EMMS \n\t" // DONE
2939
2940 : "=c" (dummy_value_c), // output regs (dummy)
2941 "=S" (dummy_value_S),
2942 "=D" (dummy_value_D)
2943
2944 : "0" (width_mmx), // ecx // input regs
2945 "1" (sptr), // esi/rsi
2946 "2" (dp) // edi/rdi
2947
2948 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
2949 : "%mm0", "%mm1" // clobber list
2950 #endif
2951 );
2952 }
2953
2954 sptr -= (width_mmx*2 - 2); // sign fixed
2955 dp -= (width_mmx*16 - 2); // sign fixed
2956 for (i = width; i; i--)
2957 {
2958 png_byte v[8];
2959 int j;
2960 sptr -= 2;
2961 png_memcpy(v, sptr, 2);
2962 for (j = 0; j < png_pass_inc[pass]; j++)
2963 {
2964 dp -= 2;
2965 png_memcpy(dp, v, 2);
2966 }
2967 }
2968 }
2969 } /* end of pixel_bytes == 2 */
2970
2971 //--------------------------------------------------------------
2972 else if (pixel_bytes == BPP8)
2973 {
2974 // GRR TEST: should work, but needs testing (special 64-bit version of rpng2?)
2975 // GRR NOTE: no need to combine passes here!
2976 if (((pass == 4) || (pass == 5)) && width)
2977 {
2978 // source is 8-byte RRGGBBAA
2979 // dest is 16-byte RRGGBBAA RRGGBBAA
2980 __asm__ __volatile__ (
2981 "sub $8, %2 \n\t" // start of last block
2982
2983 ".loop8_pass4: \n\t"
2984 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
2985 "movq %%mm0, (%2) \n\t"
2986 "sub $8, %1 \n\t"
2987 "movq %%mm0, 8(%2) \n\t"
2988 "sub $16, %2 \n\t"
2989 "decl %%ecx \n\t"
2990 "jnz .loop8_pass4 \n\t"
2991 "EMMS \n\t" // DONE
2992
2993 : "=c" (dummy_value_c), // output regs (dummy)
2994 "=S" (dummy_value_S),
2995 "=D" (dummy_value_D)
2996
2997 : "0" (width), // ecx // input regs
2998 "1" (sptr), // esi/rsi
2999 "2" (dp) // edi/rdi
3000
3001 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3002 : "%mm0" // clobber list
3003 #endif
3004 );
3005 }
3006 else if (((pass == 2) || (pass == 3)) && width)
3007 {
3008 // source is 8-byte RRGGBBAA
3009 // dest is 32-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA
3010 // (recall that expansion is _in place_: sptr and dp
3011 // both point at locations within same row buffer)
3012 __asm__ __volatile__ (
3013 "sub $24, %2 \n\t" // start of last block
3014
3015 ".loop8_pass2: \n\t"
3016 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
3017 "movq %%mm0, (%2) \n\t"
3018 "movq %%mm0, 8(%2) \n\t"
3019 "movq %%mm0, 16(%2) \n\t"
3020 "sub $8, %1 \n\t"
3021 "movq %%mm0, 24(%2) \n\t"
3022 "sub $32, %2 \n\t"
3023 "decl %%ecx \n\t"
3024 "jnz .loop8_pass2 \n\t"
3025 "EMMS \n\t" // DONE
3026
3027 : "=c" (dummy_value_c), // output regs (dummy)
3028 "=S" (dummy_value_S),
3029 "=D" (dummy_value_D)
3030
3031 : "0" (width), // ecx // input regs
3032 "1" (sptr), // esi/rsi
3033 "2" (dp) // edi/rdi
3034
3035 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
3036 : "%mm0" // clobber list
3037 #endif
3038 );
3039 }
3040 else if (width) // && ((pass == 0) || (pass == 1))
3041 {
3042 // source is 8-byte RRGGBBAA
3043 // dest is 64-byte RRGGBBAA RRGGBBAA RRGGBBAA RRGGBBAA ...
3044 __asm__ __volatile__ (
3045 "sub $56, %2 \n\t" // start of last block
3046
3047 ".loop8_pass0: \n\t"
3048 "movq (%1), %%mm0 \n\t" // 7 6 5 4 3 2 1 0
3049 "movq %%mm0, (%2) \n\t"
3050 "movq %%mm0, 8(%2) \n\t"
3051 "movq %%mm0, 16(%2) \n\t"
3052 "movq %%mm0, 24(%2) \n\t"
3053 "movq %%mm0, 32(%2) \n\t"
3054 "movq %%mm0, 40(%2) \n\t"
3055 "movq %%mm0, 48(%2) \n\t"
3056 "sub $8, %1 \n\t"
3057 "movq %%mm0, 56(%2) \n\t"
3058 "sub $64, %2 \n\t"
3059 "decl %%ecx \n\t"
3060 "jnz .loop8_pass0 \n\t"
3061 "EMMS \n\t" // DONE
3062
3063 : "=c" (dummy_value_c), // output regs (dummy)
3064 "=S" (dummy_value_S),
3065 "=D" (dummy_value_D)
3066
3067 : "0" (width), // ecx // input regs
3068 "1" (sptr), // esi/rsi
3069 "2" (dp) // edi/rdi
3070
3071 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
3072 : "%mm0" // clobber list
3073 #endif
3074 );
3075 }
3076 } /* end of pixel_bytes == 8 */
3077
3078 //--------------------------------------------------------------
3079 else if (pixel_bytes == BPP6) // why no MMX for this case?
3080 {
3081 for (i = width; i; i--)
3082 {
3083 png_byte v[8];
3084 int j;
3085 png_memcpy(v, sptr, BPP6);
3086 for (j = 0; j < png_pass_inc[pass]; j++)
3087 {
3088 png_memcpy(dp, v, BPP6);
3089 dp -= BPP6;
3090 }
3091 sptr -= BPP6;
3092 }
3093 } /* end of pixel_bytes == 6 */
3094
3095 //--------------------------------------------------------------
3096 else
3097 {
3098 // ERROR: SHOULD NEVER BE REACHED
3099 #if defined(PNG_DEBUG)
3100 png_debug(1, "Internal libpng logic error (GCC "
3101 "png_do_read_interlace() _mmx_supported)\n");
3102 #endif
3103 }
3104
3105 } // end of _mmx_supported ========================================
3106
3107 else /* MMX not supported: use modified C code - takes advantage
3108 * of inlining of png_memcpy for a constant */
3109 {
3110 if (pixel_bytes == BPP3)
3111 {
3112 for (i = width; i; i--)
3113 {
3114 png_byte v[8];
3115 int j;
3116 png_memcpy(v, sptr, BPP3);
3117 for (j = 0; j < png_pass_inc[pass]; j++)
3118 {
3119 png_memcpy(dp, v, BPP3);
3120 dp -= BPP3;
3121 }
3122 sptr -= BPP3;
3123 }
3124 }
3125 else if (pixel_bytes == BPP4)
3126 {
3127 for (i = width; i; i--)
3128 {
3129 png_byte v[8];
3130 int j;
3131 png_memcpy(v, sptr, BPP4);
3132 for (j = 0; j < png_pass_inc[pass]; j++)
3133 {
3134 #if defined(PNG_DEBUG) && defined(PNG_1_0_X) // row_buf_size gone in 1.2.x
3135 if (dp < row || dp+3 > row+png_ptr->row_buf_size)
3136 {
3137 printf("dp out of bounds: row=%10p, dp=%10p, "
3138 "rp=%10p\n", row, dp, row+png_ptr->row_buf_size);
3139 printf("row_buf_size=%lu\n", png_ptr->row_buf_size);
3140 }
3141 #endif
3142 png_memcpy(dp, v, BPP4);
3143 dp -= BPP4;
3144 }
3145 sptr -= BPP4;
3146 }
3147 }
3148 else if (pixel_bytes == 1)
3149 {
3150 for (i = width; i; i--)
3151 {
3152 int j;
3153 for (j = 0; j < png_pass_inc[pass]; j++)
3154 {
3155 *dp-- = *sptr;
3156 }
3157 --sptr;
3158 }
3159 }
3160 else if (pixel_bytes == BPP2)
3161 {
3162 for (i = width; i; i--)
3163 {
3164 png_byte v[8];
3165 int j;
3166 png_memcpy(v, sptr, BPP2);
3167 for (j = 0; j < png_pass_inc[pass]; j++)
3168 {
3169 png_memcpy(dp, v, BPP2);
3170 dp -= BPP2;
3171 }
3172 sptr -= BPP2;
3173 }
3174 }
3175 else if (pixel_bytes == BPP6)
3176 {
3177 for (i = width; i; i--)
3178 {
3179 png_byte v[8];
3180 int j;
3181 png_memcpy(v, sptr, BPP6);
3182 for (j = 0; j < png_pass_inc[pass]; j++)
3183 {
3184 png_memcpy(dp, v, BPP6);
3185 dp -= BPP6;
3186 }
3187 sptr -= BPP6;
3188 }
3189 }
3190 else if (pixel_bytes == BPP8)
3191 {
3192 for (i = width; i; i--)
3193 {
3194 png_byte v[8];
3195 int j;
3196 png_memcpy(v, sptr, BPP8);
3197 for (j = 0; j < png_pass_inc[pass]; j++)
3198 {
3199 png_memcpy(dp, v, BPP8);
3200 dp -= BPP8;
3201 }
3202 sptr -= BPP8;
3203 }
3204 }
3205 else
3206 {
3207 // ERROR: SHOULD NEVER BE REACHED
3208 #if defined(PNG_DEBUG)
3209 png_debug(1, "Internal libpng logic error (GCC "
3210 "png_do_read_interlace() !_mmx_supported)\n");
3211 #endif
3212 }
3213
3214 } /* end if (MMX not supported) */
3215 break;
3216 } /* end default (8-bit or larger) */
3217 } /* end switch (row_info->pixel_depth) */
3218
3219 row_info->width = final_width;
3220
3221 row_info->rowbytes = PNG_ROWBYTES(row_info->pixel_depth,final_width);
3222 }
3223
3224 } /* end png_do_read_interlace() */
3225
3226 #endif /* PNG_HAVE_MMX_READ_INTERLACE */
3227 #endif /* PNG_READ_INTERLACING_SUPPORTED */
3228
3229
3230
3231 #if defined(PNG_HAVE_MMX_READ_FILTER_ROW)
3232 #if defined(PNG_MMX_READ_FILTER_AVG_SUPPORTED)
3233
3234 //===========================================================================//
3235 // //
3236 // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ A V G //
3237 // //
3238 //===========================================================================//
3239
3240 // Optimized code for PNG Average filter decoder
3241
3242 static void /* PRIVATE */
png_read_filter_row_mmx_avg(png_row_infop row_info,png_bytep row,png_bytep prev_row)3243 png_read_filter_row_mmx_avg(png_row_infop row_info, png_bytep row,
3244 png_bytep prev_row)
3245 {
3246 unsigned FullLength, MMXLength; // png_uint_32 is actually 64-bit on x86-64
3247 int bpp;
3248 int dummy_value_a;
3249 int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error
3250 int dummy_value_d;
3251 png_bytep dummy_value_S;
3252 png_bytep dummy_value_D;
3253 int diff; // __attribute__((used));
3254
3255 bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel
3256 FullLength = row_info->rowbytes; // number of bytes to filter
3257
3258 __asm__ __volatile__ (
3259 "avg_top: \n\t"
3260 SAVE_GOT_ebx
3261 SAVE_r15
3262 SAVE_ebp
3263 // initialize address pointers and offset
3264 //pre "movl row, %5 \n\t" // edi/rdi: ptr to Avg(x)
3265 "xorl %%ebx, %%ebx \n\t" // ebx: x
3266 //pre "movl prev_row, %4 \n\t" // esi/rsi: ptr to Prior(x)
3267 "mov %5, " PDX " \n\t" // copy of row ptr...
3268 //pre "subl bpp, " PDX " \n\t" // (bpp is preloaded into ecx)
3269 "sub " PCX "," PDX " \n\t" // edx/rdx: ptr to Raw(x-bpp)
3270 //pre "movl FullLength, %%eax \n\t" // bring in via eax...
3271 SAVE_FullLength // ...but store for later use
3272 "xorl %%eax, %%eax \n\t"
3273
3274 // Compute the Raw value for the first bpp bytes
3275 // Raw(x) = Avg(x) + (Prior(x)/2)
3276 "avg_rlp: \n\t"
3277 "movb (%4," PBX ",), %%al \n\t" // load al with Prior(x)
3278 "incl %%ebx \n\t"
3279 "shrb %%al \n\t" // divide by 2
3280 "addb -1(%5," PBX ",), %%al \n\t" // add Avg(x); -1 to offset inc ebx
3281 //pre "cmpl bpp, %%ebx \n\t" // (bpp is preloaded into ecx)
3282 "cmpl %%ecx, %%ebx \n\t"
3283 "movb %%al, -1(%5," PBX ",) \n\t" // write Raw(x); -1 to offset inc ebx
3284 "jb avg_rlp \n\t" // mov does not affect flags
3285
3286 // get # of bytes to alignment (32-bit mask _would_ be good enough
3287 // [computing delta], but 32-bit ops are zero-extended on 64-bit, argh)
3288 // (if swapped edx and ebp, could do 8-bit or 16-bit mask...FIXME?)
3289 "mov %5, " PBP " \n\t" // take start of row
3290 "add " PBX "," PBP " \n\t" // add bpp
3291 "add $0xf, " PBP " \n\t" // add 7+8 to incr past alignment bdry
3292 // "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary (32-bit!)
3293 CLEAR_BOTTOM_3_BITS PBP "\n\t" // mask to alignment boundary
3294 "sub %5, " PBP " \n\t" // subtract row ptr again => ebp =
3295 "jz avg_go \n\t" // target value of ebx at alignment
3296
3297 "xorl %%ecx, %%ecx \n\t"
3298
3299 // fix alignment
3300 // Compute the Raw value for the bytes up to the alignment boundary
3301 // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3302 "avg_lp1: \n\t"
3303 "xorl %%eax, %%eax \n\t"
3304 "movb (%4," PBX ",), %%cl \n\t" // load cl with Prior(x)
3305 "movb (" PDX "," PBX ",), %%al \n\t" // load al with Raw(x-bpp)
3306 "addw %%cx, %%ax \n\t"
3307 "incl %%ebx \n\t"
3308 "shrw %%ax \n\t" // divide by 2
3309 "addb -1(%5," PBX ",), %%al \n\t" // add Avg(x); -1 to offset inc ebx
3310 "cmpl %%ebp, %%ebx \n\t" // check if at alignment boundary
3311 "movb %%al, -1(%5," PBX ",) \n\t" // write Raw(x); -1 to offset inc ebx
3312 "jb avg_lp1 \n\t" // repeat until at alignment boundary
3313
3314 "avg_go: \n\t"
3315 RESTORE_FullLength "%%eax \n\t" // FullLength -> eax
3316 "movl %%eax, %%ecx \n\t" // copy -> ecx
3317 "subl %%ebx, %%eax \n\t" // subtract alignment fix
3318 "andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8
3319 "subl %%eax, %%ecx \n\t" // sub over-bytes from original length
3320 //out "movl %%ecx, MMXLength \n\t"
3321 "movl %%ebp, %%eax \n\t" // ebp = diff, but no reg constraint(?)
3322 RESTORE_ebp // (could swap ebp and edx functions)
3323 RESTORE_r15
3324 RESTORE_GOT_ebx
3325
3326 // "There is no way for you to specify that an input operand is modified
3327 // without also specifying it as an output operand." [makes sense]
3328
3329 // "Unless an output operand has the `&' constraint modifier, GCC may
3330 // allocate it in the same register as an unrelated input operand, on the
3331 // assumption the inputs are consumed before the outputs are produced."
3332 // [trying to _force_ this]
3333
3334 // "`=' Means that this operand is write-only for this instruction:
3335 // the previous value is discarded and replaced by output data."
3336 // [operand == variable name, presumably]
3337
3338 // output regs
3339 // these are operands 0-1 (originally 0-3):
3340 : "=c" (MMXLength), // %0 -> %0
3341 "=a" (diff) // %3 -> %1
3342 // "=S" (dummy_value_S), // %1 -> GONE
3343 // "=D" (dummy_value_D), // %2 -> GONE
3344
3345 // input regs
3346 // these are operands 2-5 (originally 4-7); two of their constraints say
3347 // they must go in same places as operands 0-1 (originally 0-3) above:
3348 : "0" (bpp), // %4 -> %2 ecx
3349 "1" (FullLength), // %7 -> %3 eax
3350 "S" (prev_row), // %5 -> %4 esi/rsi
3351 "D" (row) // %6 -> %5 edi/rdi
3352
3353 : "%edx" // clobber list
3354 _CLOBBER_r15
3355 _CLOBBER_ebp
3356 _CLOBBER_GOT_ebx
3357 );
3358
3359 // now do the math for the rest of the row
3360 switch (bpp)
3361 {
3362 case 3:
3363 {
3364 // _ShiftBpp = 24; // == 3 * 8
3365 // _ShiftRem = 40; // == 64 - 24
3366
3367 __asm__ __volatile__ (
3368 // re-init address pointers and offset
3369 LOAD_GOT_rbp
3370 "movq " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 -> mm7
3371 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to
3372 // alignment boundary
3373 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // [interleave for parallel.?]
3374 // preload "movl row, %1 \n\t" // edi: Avg(x)
3375 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4
3376 // preload "movl prev_row, %0 \n\t" // esi: Prior(x)
3377 RESTORE_rbp
3378
3379 // prime the pump: load the first Raw(x-bpp) data set
3380 "movq -8(%1," PCX ",), %%mm2 \n\t"// load previous aligned 8 bytes
3381 // (correct pos. in loop below)
3382 "avg_3lp: \n\t"
3383 "movq (%1," PCX ",), %%mm0 \n\t" // load mm0 with Avg(x)
3384 "movq %%mm5, %%mm3 \n\t"
3385 "psrlq $40, %%mm2 \n\t" // correct position Raw(x-bpp)
3386 // data
3387 "movq (%0," PCX ",), %%mm1 \n\t" // load mm1 with Prior(x)
3388 "movq %%mm7, %%mm6 \n\t"
3389 "pand %%mm1, %%mm3 \n\t" // get lsb for each prevrow byte
3390 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3391 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3392 // byte
3393 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3394 // each byte
3395 // add 1st active group (Raw(x-bpp)/2) to average with LBCarry
3396 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3397 // LBCarrys
3398 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3399 // where both lsb's were == 1
3400 // (valid only for active group)
3401 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3402 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3403 // byte
3404 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2
3405 // for each byte
3406 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 1
3407 // bytes to add to Avg
3408 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3409 // Avg for each Active byte
3410 // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3411 "psllq $24, %%mm6 \n\t" // shift the mm6 mask to cover
3412 // bytes 3-5
3413 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3414 "psllq $24, %%mm2 \n\t" // shift data to pos. correctly
3415 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3416 // LBCarrys
3417 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3418 // where both lsb's were == 1
3419 // (valid only for active group)
3420 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3421 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3422 // byte
3423 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2
3424 // for each byte
3425 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3426 // bytes to add to Avg
3427 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3428 // Avg for each Active byte
3429
3430 // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
3431 "psllq $24, %%mm6 \n\t" // shift mm6 mask to cover last
3432 // two bytes
3433 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3434 "psllq $24, %%mm2 \n\t" // shift data to pos. correctly
3435 // Data need be shifted only once here to
3436 // get the correct x-bpp offset.
3437 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3438 // LBCarrys
3439 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3440 // where both
3441 // lsb's were == 1 (only valid for active group)
3442 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3443 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3444 // byte
3445 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to Raw(x-bpp)/2
3446 // for each byte
3447 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3448 // bytes to add to Avg
3449 "addl $8, %%ecx \n\t"
3450 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3451 // Avg for each Active byte
3452 // now ready to write back to memory
3453 "movq %%mm0, -8(%1," PCX ",) \n\t"
3454 // move updated Raw(x) to use as Raw(x-bpp) for next loop
3455 "cmpl %%eax, %%ecx \n\t" // MMXLength
3456 "movq %%mm0, %%mm2 \n\t" // mov updated Raw(x) to mm2
3457 "jb avg_3lp \n\t"
3458
3459 : "=S" (dummy_value_S), // output regs (dummy)
3460 "=D" (dummy_value_D),
3461 "=c" (dummy_value_c),
3462 "=a" (dummy_value_a)
3463
3464 : "0" (prev_row), // esi/rsi // input regs
3465 "1" (row), // edi/rdi
3466 "2" (diff), // ecx
3467 "3" (MMXLength) // eax
3468
3469 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3470 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
3471 , "%mm4", "%mm5", "%mm6", "%mm7"
3472 #endif
3473 );
3474 }
3475 break; // end 3 bpp
3476
3477 case 4: // formerly shared with 6 bpp case via _ShiftBpp and _ShiftRem,
3478 { // but loop uses all 8 MMX regs, and psrlq/psllq require 64-bit
3479 // mem (PIC/.so problems), MMX reg (none left), or immediate
3480 // _ShiftBpp = bpp << 3; // 32 (psllq)
3481 // _ShiftRem = 64 - _ShiftBpp; // 32 (psrlq)
3482
3483 __asm__ __volatile__ (
3484 LOAD_GOT_rbp
3485 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4
3486 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5
3487 // re-init address pointers and offset
3488 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to
3489 // alignment boundary
3490 "movq " AMASK0_8_0 ", %%mm7 \n\t" // _amask0_8_0 -> mm7
3491 RESTORE_rbp
3492
3493 // ... and clear all bytes except for 1st active group
3494 // preload "movl row, %1 \n\t" // edi: Avg(x)
3495 "psrlq $32, %%mm7 \n\t" // was _ShiftRem
3496 // preload "movl prev_row, %0 \n\t" // esi: Prior(x)
3497 "movq %%mm7, %%mm6 \n\t"
3498 "psllq $32, %%mm6 \n\t" // mask for 2nd active group
3499
3500 // prime the pump: load the first Raw(x-bpp) data set
3501 "movq -8(%1," PCX ",), %%mm2 \n\t" // load previous aligned 8 bytes
3502 // (we correct pos. in loop below)
3503 "avg_4lp: \n\t"
3504 "movq (%1," PCX ",), %%mm0 \n\t"
3505 "psrlq $32, %%mm2 \n\t" // shift data to pos. correctly
3506 "movq (%0," PCX ",), %%mm1 \n\t"
3507 // add (Prev_row/2) to average
3508 "movq %%mm5, %%mm3 \n\t"
3509 "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3510 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3511 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3512 // byte
3513 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3514 // each byte
3515 // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3516 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3517 // LBCarrys
3518 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3519 // where both
3520 // lsb's were == 1 (only valid for active group)
3521 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3522 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3523 // byte
3524 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3525 // for each byte
3526 "pand %%mm7, %%mm2 \n\t" // leave only Active Group 1
3527 // bytes to add to Avg
3528 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg
3529 // for each Active
3530 // byte
3531 // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3532 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3533 "psllq $32, %%mm2 \n\t" // shift data to pos. correctly
3534 "addl $8, %%ecx \n\t"
3535 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3536 // LBCarrys
3537 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3538 // where both
3539 // lsb's were == 1 (only valid for active group)
3540 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3541 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3542 // byte
3543 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3544 // for each byte
3545 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3546 // bytes to add to Avg
3547 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3548 // Avg for each Active byte
3549 "cmpl %%eax, %%ecx \n\t" // MMXLength
3550 // now ready to write back to memory
3551 "movq %%mm0, -8(%1," PCX ",) \n\t"
3552 // prep Raw(x-bpp) for next loop
3553 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3554 "jb avg_4lp \n\t"
3555
3556 : "=S" (dummy_value_S), // output regs (dummy)
3557 "=D" (dummy_value_D),
3558 "=c" (dummy_value_c),
3559 "=a" (dummy_value_a)
3560
3561 : "0" (prev_row), // esi/rsi // input regs
3562 "1" (row), // edi/rdi
3563 "2" (diff), // ecx
3564 "3" (MMXLength) // eax
3565
3566 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3567 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
3568 , "%mm4", "%mm5", "%mm6", "%mm7"
3569 #endif
3570 );
3571 }
3572 break; // end 4 bpp
3573
3574 case 1:
3575 {
3576 __asm__ __volatile__ (
3577 // re-init address pointers and offset
3578 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to align. bdry
3579 // preload "movl row, %1 \n\t" // edi/rdi: Avg(x)
3580 // preload "movl FullLength, %%eax \n\t"
3581 "cmpl %%eax, %%ecx \n\t" // test if offset at end of array
3582 "jnb avg_1end \n\t"
3583
3584 SAVE_ebp
3585
3586 // do Avg decode for remaining bytes
3587 // preload "movl prev_row, %0 \n\t" // esi/rsi: Prior(x)
3588 "mov %1, " PBP " \n\t" // copy of row pointer...
3589 "dec " PBP " \n\t" // ebp/rbp: Raw(x-bpp)
3590 "xorl %%edx, %%edx \n\t" // zero edx before using dl & dx
3591 // in loop below
3592 SAVE_GOT_ebx
3593
3594 "avg_1lp: \n\t"
3595 // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3596 "xorl %%ebx, %%ebx \n\t"
3597 "movb (%0," PCX ",), %%dl \n\t" // load dl with Prior(x)
3598 "movb (" PBP "," PCX ",), %%bl \n\t" // load bl with Raw(x-bpp)
3599 "addw %%dx, %%bx \n\t"
3600 "incl %%ecx \n\t"
3601 "shrw %%bx \n\t" // divide by 2
3602 "addb -1(%1," PCX ",), %%bl \n\t" // add Avg(x); -1 to offset
3603 // inc ecx
3604 "cmpl %%eax, %%ecx \n\t" // check if at end of array
3605 "movb %%bl, -1(%1," PCX ",) \n\t" // write back Raw(x);
3606 // mov does not affect flags; -1 to offset inc ecx
3607 "jb avg_1lp \n\t"
3608
3609 RESTORE_GOT_ebx
3610 RESTORE_ebp
3611
3612 "avg_1end: \n\t"
3613
3614 : "=S" (dummy_value_S), // output regs (dummy)
3615 "=D" (dummy_value_D),
3616 "=c" (dummy_value_c),
3617 "=a" (dummy_value_a)
3618
3619 : "0" (prev_row), // esi/rsi // input regs
3620 "1" (row), // edi/rdi
3621 "2" (diff), // ecx
3622 "3" (FullLength) // eax
3623
3624 : "%edx" // clobber list
3625 _CLOBBER_GOT_ebx
3626 _CLOBBER_ebp
3627 );
3628 }
3629 return; // end 1 bpp
3630
3631 case 2:
3632 {
3633 // _ShiftBpp = 16; // == 2 * 8
3634 // _ShiftRem = 48; // == 64 - _ShiftBpp
3635
3636 __asm__ __volatile__ (
3637 LOAD_GOT_rbp
3638 // load (former) _ActiveMask
3639 "movq " AMASK6_2_0 ", %%mm7 \n\t" // _amask6_2_0 -> mm7
3640 // re-init address pointers and offset
3641 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to
3642 // alignment boundary
3643 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5
3644 // preload "movl row, %1 \n\t" // edi: Avg(x)
3645 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4
3646 // preload "movl prev_row, %0 \n\t" // esi: Prior(x)
3647 RESTORE_rbp
3648
3649 // prime the pump: load the first Raw(x-bpp) data set
3650 "movq -8(%1," PCX ",), %%mm2 \n\t" // load previous aligned 8 bytes
3651 // (we correct pos. in loop below)
3652 "avg_2lp: \n\t"
3653 "movq (%1," PCX ",), %%mm0 \n\t"
3654 "psrlq $48, %%mm2 \n\t" // shift data to pos. correctly
3655 "movq (%0," PCX ",), %%mm1 \n\t" // (GRR BUGFIX: was psllq)
3656 // add (Prev_row/2) to average
3657 "movq %%mm5, %%mm3 \n\t"
3658 "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3659 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3660 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3661 // byte
3662 "movq %%mm7, %%mm6 \n\t"
3663 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3664 // each byte
3665
3666 // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3667 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3668 // LBCarrys
3669 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3670 // where both
3671 // lsb's were == 1 (only valid
3672 // for active group)
3673 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3674 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3675 // byte
3676 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3677 // for each byte
3678 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 1
3679 // bytes to add to Avg
3680 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg
3681 // for each Active byte
3682
3683 // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3684 "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover
3685 // bytes 2 & 3
3686 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3687 "psllq $16, %%mm2 \n\t" // shift data to pos. correctly
3688 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3689 // LBCarrys
3690 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3691 // where both
3692 // lsb's were == 1 (only valid
3693 // for active group)
3694 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3695 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3696 // byte
3697 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3698 // for each byte
3699 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3700 // bytes to add to Avg
3701 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3702 // Avg for each Active byte
3703
3704 // add 3rd active group (Raw(x-bpp)/2) to average with _LBCarry
3705 "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover
3706 // bytes 4 & 5
3707 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3708 "psllq $16, %%mm2 \n\t" // shift data to pos. correctly
3709 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3710 // LBCarrys
3711 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3712 // where both lsb's were == 1
3713 // (only valid for active group)
3714 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3715 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3716 // byte
3717 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3718 // for each byte
3719 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3720 // bytes to add to Avg
3721 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3722 // Avg for each Active byte
3723
3724 // add 4th active group (Raw(x-bpp)/2) to average with _LBCarry
3725 "psllq $16, %%mm6 \n\t" // shift the mm6 mask to cover
3726 // bytes 6 & 7
3727 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3728 "psllq $16, %%mm2 \n\t" // shift data to pos. correctly
3729 "addl $8, %%ecx \n\t"
3730 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3731 // LBCarrys
3732 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3733 // where both
3734 // lsb's were == 1 (only valid
3735 // for active group)
3736 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3737 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3738 // byte
3739 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3740 // for each byte
3741 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3742 // bytes to add to Avg
3743 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3744 // Avg for each Active byte
3745 "cmpl %%eax, %%ecx \n\t" // MMXLength
3746 // now ready to write back to memory
3747 "movq %%mm0, -8(%1," PCX ",) \n\t"
3748 // prep Raw(x-bpp) for next loop
3749 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3750 "jb avg_2lp \n\t"
3751
3752 : "=S" (dummy_value_S), // output regs (dummy)
3753 "=D" (dummy_value_D),
3754 "=c" (dummy_value_c),
3755 "=a" (dummy_value_a)
3756
3757 : "0" (prev_row), // esi/rsi // input regs
3758 "1" (row), // edi/rdi
3759 "2" (diff), // ecx
3760 "3" (MMXLength) // eax
3761
3762 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3763 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
3764 , "%mm4", "%mm5", "%mm6", "%mm7"
3765 #endif
3766 );
3767 }
3768 break; // end 2 bpp
3769
3770 case 6: // formerly shared with 4 bpp case (see comments there)
3771 {
3772 // _ShiftBpp = bpp << 3; // 48 (psllq)
3773 // _ShiftRem = 64 - _ShiftBpp; // 16 (psrlq)
3774
3775 __asm__ __volatile__ (
3776 LOAD_GOT_rbp
3777 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4
3778 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // _LBCarryMask -> mm5
3779 // re-init address pointers and offset
3780 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to
3781 // alignment boundary
3782 "movq " AMASK0_8_0 ", %%mm7 \n\t" // _amask0_8_0 -> mm7
3783 RESTORE_rbp
3784
3785 // ... and clear all bytes except for 1st active group
3786 // preload "movl row, %1 \n\t" // edi: Avg(x)
3787 "psrlq $16, %%mm7 \n\t"
3788 // preload "movl prev_row, %0 \n\t" // esi: Prior(x)
3789 "movq %%mm7, %%mm6 \n\t"
3790 "psllq $48, %%mm6 \n\t" // mask for 2nd active group
3791
3792 // prime the pump: load the first Raw(x-bpp) data set
3793 "movq -8(%1," PCX ",), %%mm2 \n\t" // load previous aligned 8 bytes
3794 // (we correct pos. in loop below)
3795 "avg_6lp: \n\t"
3796 "movq (%1," PCX ",), %%mm0 \n\t"
3797 "psrlq $16, %%mm2 \n\t" // shift data to pos. correctly
3798 "movq (%0," PCX ",), %%mm1 \n\t"
3799 // add (Prev_row/2) to average
3800 "movq %%mm5, %%mm3 \n\t"
3801 "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3802 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3803 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7 of each
3804 // byte
3805 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg for
3806 // each byte
3807 // add 1st active group (Raw(x-bpp)/2) to average with _LBCarry
3808 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3809 // LBCarrys
3810 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3811 // where both
3812 // lsb's were == 1 (only valid for active group)
3813 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3814 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3815 // byte
3816 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3817 // for each byte
3818 "pand %%mm7, %%mm2 \n\t" // leave only Active Group 1
3819 // bytes to add to Avg
3820 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to Avg
3821 // for each Active
3822 // byte
3823 // add 2nd active group (Raw(x-bpp)/2) to average with _LBCarry
3824 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3825 "psllq $48, %%mm2 \n\t" // shift data to pos. correctly
3826 "addl $8, %%ecx \n\t"
3827 "movq %%mm3, %%mm1 \n\t" // now use mm1 for getting
3828 // LBCarrys
3829 "pand %%mm2, %%mm1 \n\t" // get LBCarrys for each byte
3830 // where both
3831 // lsb's were == 1 (only valid for active group)
3832 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3833 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7 of each
3834 // byte
3835 "paddb %%mm1, %%mm2 \n\t" // add LBCarrys to (Raw(x-bpp)/2)
3836 // for each byte
3837 "pand %%mm6, %%mm2 \n\t" // leave only Active Group 2
3838 // bytes to add to Avg
3839 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) + LBCarrys to
3840 // Avg for each Active byte
3841 "cmpl %%eax, %%ecx \n\t" // MMXLength
3842 // now ready to write back to memory
3843 "movq %%mm0, -8(%1," PCX ",) \n\t"
3844 // prep Raw(x-bpp) for next loop
3845 "movq %%mm0, %%mm2 \n\t" // mov updated Raws to mm2
3846 "jb avg_6lp \n\t"
3847
3848 : "=S" (dummy_value_S), // output regs (dummy)
3849 "=D" (dummy_value_D),
3850 "=c" (dummy_value_c),
3851 "=a" (dummy_value_a)
3852
3853 : "0" (prev_row), // esi/rsi // input regs
3854 "1" (row), // edi/rdi
3855 "2" (diff), // ecx
3856 "3" (MMXLength) // eax
3857
3858 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3859 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
3860 , "%mm4", "%mm5", "%mm6", "%mm7"
3861 #endif
3862 );
3863 }
3864 break; // end 6 bpp
3865
3866 case 8:
3867 {
3868 __asm__ __volatile__ (
3869 // re-init address pointers and offset
3870 // preload "movl diff, %%ecx \n\t" // ecx: x = offset to
3871 // alignment boundary
3872 LOAD_GOT_rbp
3873 "movq " LB_CARRY_MASK ", %%mm5 \n\t" // [interleave for parallel.?]
3874 // preload "movl row, %1 \n\t" // edi: Avg(x)
3875 "movq " HB_CLEAR_MASK ", %%mm4 \n\t" // _HBClearMask -> mm4
3876 // preload "movl prev_row, %0 \n\t" // esi: Prior(x)
3877 RESTORE_rbp
3878
3879 // prime the pump: load the first Raw(x-bpp) data set
3880 "movq -8(%1," PCX ",), %%mm2 \n\t" // load previous aligned 8 bytes
3881 // (NO NEED to correct pos. in loop below)
3882
3883 "avg_8lp: \n\t"
3884 "movq (%1," PCX ",), %%mm0 \n\t"
3885 "movq %%mm5, %%mm3 \n\t"
3886 "movq (%0," PCX ",), %%mm1 \n\t"
3887 "addl $8, %%ecx \n\t"
3888 "pand %%mm1, %%mm3 \n\t" // get lsb for each prev_row byte
3889 "psrlq $1, %%mm1 \n\t" // divide prev_row bytes by 2
3890 "pand %%mm2, %%mm3 \n\t" // get LBCarrys for each byte
3891 // where both lsb's were == 1
3892 "psrlq $1, %%mm2 \n\t" // divide raw bytes by 2
3893 "pand %%mm4, %%mm1 \n\t" // clear invalid bit 7, each byte
3894 "paddb %%mm3, %%mm0 \n\t" // add LBCarrys to Avg, each byte
3895 "pand %%mm4, %%mm2 \n\t" // clear invalid bit 7, each byte
3896 "paddb %%mm1, %%mm0 \n\t" // add (Prev_row/2) to Avg, each
3897 "paddb %%mm2, %%mm0 \n\t" // add (Raw/2) to Avg for each
3898 "cmpl %%eax, %%ecx \n\t" // MMXLength
3899 "movq %%mm0, -8(%1," PCX ",) \n\t"
3900 "movq %%mm0, %%mm2 \n\t" // reuse as Raw(x-bpp)
3901 "jb avg_8lp \n\t"
3902
3903 : "=S" (dummy_value_S), // output regs (dummy)
3904 "=D" (dummy_value_D),
3905 "=c" (dummy_value_c),
3906 "=a" (dummy_value_a)
3907
3908 : "0" (prev_row), // esi/rsi // input regs
3909 "1" (row), // edi/rdi
3910 "2" (diff), // ecx
3911 "3" (MMXLength) // eax
3912
3913 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
3914 : "%mm0", "%mm1", "%mm2" // clobber list
3915 , "%mm3", "%mm4", "%mm5"
3916 #endif
3917 );
3918 }
3919 break; // end 8 bpp
3920
3921 default: // bpp != 1,2,3,4,6,8: doesn't exist
3922 {
3923 // ERROR: SHOULD NEVER BE REACHED
3924 #if defined(PNG_DEBUG)
3925 png_debug(1, "Internal libpng logic error (GCC "
3926 "png_read_filter_row_mmx_avg())\n");
3927 #endif
3928 }
3929 break;
3930
3931 } // end switch (bpp)
3932
3933 __asm__ __volatile__ (
3934 // MMX acceleration complete; now do clean-up
3935 // check if any remaining bytes left to decode
3936 //pre "movl FullLength, %%edx \n\t"
3937 //pre "movl MMXLength, %%eax \n\t" // eax: x == offset bytes after MMX
3938 //pre "movl row, %2 \n\t" // edi: Avg(x)
3939 "cmpl %%edx, %%eax \n\t" // test if offset at end of array
3940 "jnb avg_end \n\t"
3941
3942 SAVE_ebp
3943
3944 // do Avg decode for remaining bytes
3945 //pre "movl prev_row, %1 \n\t" // esi: Prior(x)
3946 "mov %2, " PBP " \n\t" // copy of row pointer...
3947 //pre "subl bpp, " PBP " \n\t" // (bpp is preloaded into ecx)
3948 "sub " PCX "," PBP " \n\t" // ebp: Raw(x-bpp)
3949 "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below
3950
3951 SAVE_GOT_ebx
3952
3953 "avg_lp2: \n\t"
3954 // Raw(x) = Avg(x) + ((Raw(x-bpp) + Prior(x))/2)
3955 "xorl %%ebx, %%ebx \n\t"
3956 "movb (%1," PAX ",), %%cl \n\t" // load cl with Prior(x)
3957 "movb (" PBP "," PAX ",), %%bl \n\t" // load bl with Raw(x-bpp)
3958 "addw %%cx, %%bx \n\t"
3959 "incl %%eax \n\t"
3960 "shrw %%bx \n\t" // divide by 2
3961 "addb -1(%2," PAX ",), %%bl \n\t" // add Avg(x); -1 to offset inc eax
3962 "cmpl %%edx, %%eax \n\t" // check if at end of array
3963 "movb %%bl, -1(%2," PAX ",) \n\t" // write back Raw(x) [mov does not
3964 "jb avg_lp2 \n\t" // affect flags; -1 to offset inc eax]
3965
3966 RESTORE_GOT_ebx
3967 RESTORE_ebp
3968
3969 "avg_end: \n\t"
3970 "EMMS \n\t" // end MMX; prep for poss. FP instrs.
3971
3972 : "=c" (dummy_value_c), // output regs (dummy)
3973 "=S" (dummy_value_S),
3974 "=D" (dummy_value_D),
3975 "=a" (dummy_value_a),
3976 "=d" (dummy_value_d)
3977
3978 : "0" (bpp), // ecx // input regs
3979 "1" (prev_row), // esi/rsi
3980 "2" (row), // edi/rdi
3981 "3" (MMXLength), // eax
3982 "4" (FullLength) // edx
3983
3984 CLOB_COLON_ebx_ebp // clobber list
3985 CLOBBER_GOT_ebx
3986 CLOB_COMMA_ebx_ebp
3987 CLOBBER_ebp
3988 );
3989
3990 } /* end png_read_filter_row_mmx_avg() */
3991
3992 #endif /* PNG_MMX_READ_FILTER_AVG_SUPPORTED */
3993
3994
3995
3996 #if defined(PNG_MMX_READ_FILTER_PAETH_SUPPORTED)
3997 #if defined(PNG_x86_64_USE_GOTPCREL) || defined(PNG_THREAD_UNSAFE_OK)
3998
3999 //===========================================================================//
4000 // //
4001 // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ P A E T H //
4002 // //
4003 //===========================================================================//
4004
4005 // Optimized code for PNG Paeth filter decoder
4006
4007 static void /* PRIVATE */
png_read_filter_row_mmx_paeth(png_row_infop row_info,png_bytep row,png_bytep prev_row)4008 png_read_filter_row_mmx_paeth(png_row_infop row_info, png_bytep row,
4009 png_bytep prev_row)
4010 {
4011 unsigned FullLength, MMXLength; // png_uint_32 is actually 64-bit on x86-64
4012 int bpp;
4013 int dummy_value_a;
4014 int dummy_value_c; // fix 'forbidden register 2 (cx) was spilled' error
4015 int dummy_value_d;
4016 png_charp dummy_value_S;
4017 png_charp dummy_value_D;
4018 int diff; // __attribute__((used));
4019
4020 bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel
4021 FullLength = row_info->rowbytes; // number of bytes to filter
4022
4023 __asm__ __volatile__ (
4024 SAVE_GOT_ebx
4025 SAVE_r15
4026 SAVE_ebp
4027 //pre "movl row, %2 \n\t" // edi/rdi
4028 "xorl %%ebx, %%ebx \n\t" // ebx: x offset
4029 //pre "movl prev_row, %1 \n\t" // esi/rsi
4030 "xorl %%edx, %%edx \n\t" // edx: x-bpp offset
4031 //pre "movl FullLength, %%eax \n\t" // bring in via eax...
4032 SAVE_FullLength // ...but store for later use
4033 "xorl %%eax, %%eax \n\t"
4034
4035 // Compute the Raw value for the first bpp bytes
4036 // Note: the formula works out to be always
4037 // Paeth(x) = Raw(x) + Prior(x) where x < bpp
4038 "paeth_rlp: \n\t"
4039 "movb (%2," PBX ",), %%al \n\t"
4040 "addb (%1," PBX ",), %%al \n\t"
4041 "incl %%ebx \n\t"
4042 //pre "cmpl bpp, %%ebx \n\t" (bpp is preloaded into ecx)
4043 "cmpl %%ecx, %%ebx \n\t"
4044 "movb %%al, -1(%2," PBX ",) \n\t"
4045 "jb paeth_rlp \n\t"
4046
4047 // get # of bytes to alignment (note: computing _delta_ of two pointers,
4048 // so hereafter %%ebp is sufficient even on 64-bit)
4049 "mov %2, " PBP " \n\t" // take start of row
4050 "add " PBX "," PBP " \n\t" // add bpp
4051 "add $0xf, " PBP " \n\t" // add 7+8 to incr past alignment bdry
4052 // "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary (32-bit!)
4053 CLEAR_BOTTOM_3_BITS PBP "\n\t" // mask to alignment boundary
4054 "sub %2, " PBP " \n\t" // subtract row ptr again => ebp =
4055 "jz paeth_go \n\t" // target value of ebx at alignment
4056
4057 "xorl %%ecx, %%ecx \n\t"
4058
4059 SAVE_r11_r12_r13
4060
4061 // fix alignment
4062 "paeth_lp1: \n\t"
4063 "xorl %%eax, %%eax \n\t"
4064 // pav = p - a = (a + b - c) - a = b - c
4065 "movb (%1," PBX ",), %%al \n\t" // load Prior(x) into al
4066 "movb (%1," PDX ",), %%cl \n\t" // load Prior(x-bpp) into cl
4067 "subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4068 "movl %%eax, " pa_TEMP " \n\t" // Save pav for later use
4069 "xorl %%eax, %%eax \n\t"
4070 // pbv = p - b = (a + b - c) - b = a - c
4071 "movb (%2," PDX ",), %%al \n\t" // load Raw(x-bpp) into al
4072 "subl %%ecx, %%eax \n\t" // subtract Prior(x-bpp)
4073 "movl %%eax, %%ecx \n\t"
4074 // pcv = p - c = (a + b - c) - c = (a - c) + (b - c) = pav + pbv
4075 "addl " pa_TEMP ", %%eax \n\t" // pcv = pav + pbv
4076 // pc = abs(pcv)
4077 "testl $0x80000000, %%eax \n\t"
4078 "jz paeth_pca \n\t"
4079 "negl %%eax \n\t" // reverse sign of neg values
4080
4081 "paeth_pca: \n\t"
4082 "movl %%eax, " pc_TEMP " \n\t" // save pc for later use
4083 // pb = abs(pbv)
4084 "testl $0x80000000, %%ecx \n\t"
4085 "jz paeth_pba \n\t"
4086 "negl %%ecx \n\t" // reverse sign of neg values
4087
4088 "paeth_pba: \n\t"
4089 "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use
4090 // pa = abs(pav)
4091 "movl " pa_TEMP ", %%eax \n\t"
4092 "testl $0x80000000, %%eax \n\t"
4093 "jz paeth_paa \n\t"
4094 "negl %%eax \n\t" // reverse sign of neg values
4095
4096 "paeth_paa: \n\t"
4097 "movl %%eax, " pa_TEMP " \n\t" // save pa for later use
4098 // test if pa <= pb
4099 "cmpl %%ecx, %%eax \n\t"
4100 "jna paeth_abb \n\t"
4101 // pa > pb; now test if pb <= pc
4102 "cmpl " pc_TEMP ", %%ecx \n\t"
4103 "jna paeth_bbc \n\t"
4104 // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4105 "movb (%1," PDX ",), %%cl \n\t" // load Prior(x-bpp) into cl
4106 "jmp paeth_paeth \n\t"
4107
4108 "paeth_bbc: \n\t"
4109 // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4110 "movb (%1," PBX ",), %%cl \n\t" // load Prior(x) into cl
4111 "jmp paeth_paeth \n\t"
4112
4113 "paeth_abb: \n\t"
4114 // pa <= pb; now test if pa <= pc
4115 "cmpl " pc_TEMP ", %%eax \n\t"
4116 "jna paeth_abc \n\t"
4117 // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4118 "movb (%1," PDX ",), %%cl \n\t" // load Prior(x-bpp) into cl
4119 "jmp paeth_paeth \n\t"
4120
4121 "paeth_abc: \n\t"
4122 // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4123 "movb (%2," PDX ",), %%cl \n\t" // load Raw(x-bpp) into cl
4124
4125 "paeth_paeth: \n\t"
4126 "incl %%ebx \n\t"
4127 "incl %%edx \n\t"
4128 // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4129 "addb %%cl, -1(%2," PBX ",) \n\t"
4130 "cmpl %%ebp, %%ebx \n\t"
4131 "jb paeth_lp1 \n\t"
4132
4133 RESTORE_r11_r12_r13
4134
4135 "paeth_go: \n\t"
4136 RESTORE_FullLength "%%ecx \n\t" // FullLength -> ecx
4137 "movl %%ecx, %%eax \n\t"
4138 "subl %%ebx, %%eax \n\t" // subtract alignment fix
4139 "andl $0x00000007, %%eax \n\t" // calc bytes over mult of 8
4140 "subl %%eax, %%ecx \n\t" // drop over bytes from original length
4141 //out "movl %%ecx, MMXLength \n\t"
4142 "movl %%ebp, %%eax \n\t" // ebp = diff, but no reg constraint(?)
4143 RESTORE_ebp // (could swap ebp and edx functions)
4144 RESTORE_r15
4145 RESTORE_GOT_ebx
4146
4147 : "=c" (MMXLength), // output regs
4148 "=S" (dummy_value_S),
4149 "=D" (dummy_value_D),
4150 "=a" (diff)
4151
4152 : "0" (bpp), // ecx // input regs
4153 "1" (prev_row), // esi/rsi
4154 "2" (row), // edi/rdi
4155 "3" (FullLength) // eax
4156
4157 : "%edx" // clobber list
4158 _CLOBBER_r11_r12_r13
4159 _CLOBBER_r15
4160 _CLOBBER_ebp
4161 _CLOBBER_GOT_ebx
4162 );
4163
4164 // now do the math for the rest of the row
4165 switch (bpp)
4166 {
4167 case 3:
4168 {
4169 // _ShiftBpp = 24; // == bpp * 8
4170 // _ShiftRem = 40; // == 64 - _ShiftBpp
4171
4172 __asm__ __volatile__ (
4173 LOAD_GOT_rbp
4174 // preload "movl diff, %%ecx \n\t"
4175 // preload "movl row, %1 \n\t" // edi/rdi
4176 // preload "movl prev_row, %0 \n\t" // esi/rsi
4177 "pxor %%mm0, %%mm0 \n\t"
4178
4179 // prime the pump: load the first Raw(x-bpp) data set
4180 "movq -8(%1," PCX ",), %%mm1 \n\t"
4181 "paeth_3lp: \n\t"
4182 "psrlq $40, %%mm1 \n\t" // shift last 3 bytes to 1st
4183 // 3 bytes
4184 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4185 "punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4186 "movq -8(%0," PCX ",), %%mm3 \n\t" // prep c=Prior(x-bpp) bytes
4187 "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4188 "psrlq $40, %%mm3 \n\t" // shift last 3 bytes to 1st
4189 // 3 bytes
4190 // pav = p - a = (a + b - c) - a = b - c
4191 "movq %%mm2, %%mm4 \n\t"
4192 "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4193 // pbv = p - b = (a + b - c) - b = a - c
4194 "movq %%mm1, %%mm5 \n\t"
4195 "psubw %%mm3, %%mm4 \n\t"
4196 "pxor %%mm7, %%mm7 \n\t"
4197 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4198 "movq %%mm4, %%mm6 \n\t"
4199 "psubw %%mm3, %%mm5 \n\t"
4200
4201 // pa = abs(p-a) = abs(pav)
4202 // pb = abs(p-b) = abs(pbv)
4203 // pc = abs(p-c) = abs(pcv)
4204 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4205 "paddw %%mm5, %%mm6 \n\t"
4206 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4207 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4208 "psubw %%mm0, %%mm4 \n\t"
4209 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4210 "psubw %%mm0, %%mm4 \n\t"
4211 "psubw %%mm7, %%mm5 \n\t"
4212 "pxor %%mm0, %%mm0 \n\t"
4213 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4214 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4215 "psubw %%mm7, %%mm5 \n\t"
4216 "psubw %%mm0, %%mm6 \n\t"
4217 // test pa <= pb
4218 "movq %%mm4, %%mm7 \n\t"
4219 "psubw %%mm0, %%mm6 \n\t"
4220 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4221 "movq %%mm7, %%mm0 \n\t"
4222 // use mm7 mask to merge pa & pb
4223 "pand %%mm7, %%mm5 \n\t"
4224 // use mm0 mask copy to merge a & b
4225 "pand %%mm0, %%mm2 \n\t"
4226 "pandn %%mm4, %%mm7 \n\t"
4227 "pandn %%mm1, %%mm0 \n\t"
4228 "paddw %%mm5, %%mm7 \n\t"
4229 "paddw %%mm2, %%mm0 \n\t"
4230 // test ((pa <= pb)? pa:pb) <= pc
4231 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4232 "pxor %%mm1, %%mm1 \n\t"
4233 "pand %%mm7, %%mm3 \n\t"
4234 "pandn %%mm0, %%mm7 \n\t"
4235 "paddw %%mm3, %%mm7 \n\t"
4236 "pxor %%mm0, %%mm0 \n\t"
4237 "packuswb %%mm1, %%mm7 \n\t"
4238 "movq (%0," PCX ",), %%mm3 \n\t" // load c=Prior(x-bpp)
4239 "pand " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 (was _ActiveMask)
4240 "movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1
4241 "paddb (%1," PCX ",), %%mm7 \n\t" // add Paeth predictor + Raw(x)
4242 "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4243 "movq %%mm7, (%1," PCX ",) \n\t" // write back updated value
4244 "movq %%mm7, %%mm1 \n\t" // now mm1 will be used as
4245 // Raw(x-bpp)
4246 // now do Paeth for 2nd set of bytes (3-5)
4247 "psrlq $24, %%mm2 \n\t" // load b=Prior(x) step 2
4248 "punpcklbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4249 "pxor %%mm7, %%mm7 \n\t"
4250 "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4251 // pbv = p - b = (a + b - c) - b = a - c
4252 "movq %%mm1, %%mm5 \n\t"
4253 // pav = p - a = (a + b - c) - a = b - c
4254 "movq %%mm2, %%mm4 \n\t"
4255 "psubw %%mm3, %%mm5 \n\t"
4256 "psubw %%mm3, %%mm4 \n\t"
4257 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) =
4258 // pav + pbv = pbv + pav
4259 "movq %%mm5, %%mm6 \n\t"
4260 "paddw %%mm4, %%mm6 \n\t"
4261
4262 // pa = abs(p-a) = abs(pav)
4263 // pb = abs(p-b) = abs(pbv)
4264 // pc = abs(p-c) = abs(pcv)
4265 "pcmpgtw %%mm5, %%mm0 \n\t" // create mask pbv bytes < 0
4266 "pcmpgtw %%mm4, %%mm7 \n\t" // create mask pav bytes < 0
4267 "pand %%mm5, %%mm0 \n\t" // only pbv bytes < 0 in mm0
4268 "pand %%mm4, %%mm7 \n\t" // only pav bytes < 0 in mm7
4269 "psubw %%mm0, %%mm5 \n\t"
4270 "psubw %%mm7, %%mm4 \n\t"
4271 "psubw %%mm0, %%mm5 \n\t"
4272 "psubw %%mm7, %%mm4 \n\t"
4273 "pxor %%mm0, %%mm0 \n\t"
4274 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4275 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4276 "psubw %%mm0, %%mm6 \n\t"
4277 // test pa <= pb
4278 "movq %%mm4, %%mm7 \n\t"
4279 "psubw %%mm0, %%mm6 \n\t"
4280 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4281 "movq %%mm7, %%mm0 \n\t"
4282 // use mm7 mask to merge pa & pb
4283 "pand %%mm7, %%mm5 \n\t"
4284 // use mm0 mask copy to merge a & b
4285 "pand %%mm0, %%mm2 \n\t"
4286 "pandn %%mm4, %%mm7 \n\t"
4287 "pandn %%mm1, %%mm0 \n\t"
4288 "paddw %%mm5, %%mm7 \n\t"
4289 "paddw %%mm2, %%mm0 \n\t"
4290 // test ((pa <= pb)? pa:pb) <= pc
4291 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4292 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4293 "pand %%mm7, %%mm3 \n\t"
4294 "pandn %%mm0, %%mm7 \n\t"
4295 "pxor %%mm1, %%mm1 \n\t"
4296 "paddw %%mm3, %%mm7 \n\t"
4297 "pxor %%mm0, %%mm0 \n\t"
4298 "packuswb %%mm1, %%mm7 \n\t"
4299 "movq %%mm2, %%mm3 \n\t" // load c=Prior(x-bpp) step 1
4300 "pand " AMASK5_3_0 ", %%mm7 \n\t" // _amask5_3_0 (was _ActiveMask)
4301 "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4302 "psllq $24, %%mm7 \n\t" // shift bytes to 2nd group of
4303 // 3 bytes
4304 // pav = p - a = (a + b - c) - a = b - c
4305 "movq %%mm2, %%mm4 \n\t"
4306 "paddb (%1," PCX ",), %%mm7 \n\t" // add Paeth predictor + Raw(x)
4307 "psllq $24, %%mm3 \n\t" // load c=Prior(x-bpp) step 2
4308 "movq %%mm7, (%1," PCX ",) \n\t" // write back updated value
4309 "movq %%mm7, %%mm1 \n\t"
4310 "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4311 "psllq $24, %%mm1 \n\t" // shift bytes (was _ShiftBpp)
4312 // now mm1 will be used as Raw(x-bpp)
4313 // now do Paeth for 3rd, and final, set of bytes (6-7)
4314 "pxor %%mm7, %%mm7 \n\t"
4315 "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4316 "psubw %%mm3, %%mm4 \n\t"
4317 // pbv = p - b = (a + b - c) - b = a - c
4318 "movq %%mm1, %%mm5 \n\t"
4319 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4320 "movq %%mm4, %%mm6 \n\t"
4321 "psubw %%mm3, %%mm5 \n\t"
4322 "pxor %%mm0, %%mm0 \n\t"
4323 "paddw %%mm5, %%mm6 \n\t"
4324
4325 // pa = abs(p-a) = abs(pav)
4326 // pb = abs(p-b) = abs(pbv)
4327 // pc = abs(p-c) = abs(pcv)
4328 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4329 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4330 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4331 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4332 "psubw %%mm0, %%mm4 \n\t"
4333 "psubw %%mm7, %%mm5 \n\t"
4334 "psubw %%mm0, %%mm4 \n\t"
4335 "psubw %%mm7, %%mm5 \n\t"
4336 "pxor %%mm0, %%mm0 \n\t"
4337 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4338 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4339 "psubw %%mm0, %%mm6 \n\t"
4340 // test pa <= pb
4341 "movq %%mm4, %%mm7 \n\t"
4342 "psubw %%mm0, %%mm6 \n\t"
4343 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4344 "movq %%mm7, %%mm0 \n\t"
4345 // use mm0 mask copy to merge a & b
4346 "pand %%mm0, %%mm2 \n\t"
4347 // use mm7 mask to merge pa & pb
4348 "pand %%mm7, %%mm5 \n\t"
4349 "pandn %%mm1, %%mm0 \n\t"
4350 "pandn %%mm4, %%mm7 \n\t"
4351 "paddw %%mm2, %%mm0 \n\t"
4352 "paddw %%mm5, %%mm7 \n\t"
4353 // test ((pa <= pb)? pa:pb) <= pc
4354 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4355 "pand %%mm7, %%mm3 \n\t"
4356 "pandn %%mm0, %%mm7 \n\t"
4357 "paddw %%mm3, %%mm7 \n\t"
4358 "pxor %%mm1, %%mm1 \n\t"
4359 "packuswb %%mm7, %%mm1 \n\t"
4360 // step ecx to next set of 8 bytes and repeat loop til done
4361 "addl $8, %%ecx \n\t"
4362 "pand " AMASK0_2_6 ", %%mm1 \n\t" // _amask0_2_6 (_ActiveMaskEnd)
4363 "paddb -8(%1," PCX ",), %%mm1 \n\t" // add Paeth predictor + Raw(x)
4364 "cmpl %%eax, %%ecx \n\t" // MMXLength
4365 "pxor %%mm0, %%mm0 \n\t" // pxor does not affect flags
4366 "movq %%mm1, -8(%1," PCX ",) \n\t" // write back updated value
4367 // mm1 will be used as Raw(x-bpp) next loop
4368 // mm3 ready to be used as Prior(x-bpp) next loop
4369 "jb paeth_3lp \n\t"
4370 RESTORE_rbp
4371
4372 : "=S" (dummy_value_S), // output regs (dummy)
4373 "=D" (dummy_value_D),
4374 "=c" (dummy_value_c),
4375 "=a" (dummy_value_a)
4376
4377 : "0" (prev_row), // esi/rsi // input regs
4378 "1" (row), // edi/rdi
4379 "2" (diff), // ecx
4380 "3" (MMXLength) // eax
4381
4382 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
4383 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
4384 , "%mm4", "%mm5", "%mm6", "%mm7"
4385 #endif
4386 );
4387 }
4388 break; // end 3 bpp
4389
4390 case 4:
4391 {
4392 __asm__ __volatile__ (
4393 // preload "movl diff, %%ecx \n\t"
4394 // preload "movl row, %1 \n\t" // edi/rdi
4395 // preload "movl prev_row, %0 \n\t" // esi/rsi
4396 "pxor %%mm0, %%mm0 \n\t"
4397 // prime the pump: load the first Raw(x-bpp) data set
4398 "movq -8(%1," PCX ",), %%mm1 \n\t" // only time should need to read
4399 // a=Raw(x-bpp) bytes
4400 "paeth_4lp: \n\t"
4401 // do first set of 4 bytes
4402 "movq -8(%0," PCX ",), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4403 "punpckhbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4404 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4405 "punpcklbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4406 // pav = p - a = (a + b - c) - a = b - c
4407 "movq %%mm2, %%mm4 \n\t"
4408 "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4409 // pbv = p - b = (a + b - c) - b = a - c
4410 "movq %%mm1, %%mm5 \n\t"
4411 "psubw %%mm3, %%mm4 \n\t"
4412 "pxor %%mm7, %%mm7 \n\t"
4413 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4414 "movq %%mm4, %%mm6 \n\t"
4415 "psubw %%mm3, %%mm5 \n\t"
4416 // pa = abs(p-a) = abs(pav)
4417 // pb = abs(p-b) = abs(pbv)
4418 // pc = abs(p-c) = abs(pcv)
4419 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4420 "paddw %%mm5, %%mm6 \n\t"
4421 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4422 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4423 "psubw %%mm0, %%mm4 \n\t"
4424 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4425 "psubw %%mm0, %%mm4 \n\t"
4426 "psubw %%mm7, %%mm5 \n\t"
4427 "pxor %%mm0, %%mm0 \n\t"
4428 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4429 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4430 "psubw %%mm7, %%mm5 \n\t"
4431 "psubw %%mm0, %%mm6 \n\t"
4432 // test pa <= pb
4433 "movq %%mm4, %%mm7 \n\t"
4434 "psubw %%mm0, %%mm6 \n\t"
4435 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4436 "movq %%mm7, %%mm0 \n\t"
4437 // use mm7 mask to merge pa & pb
4438 "pand %%mm7, %%mm5 \n\t"
4439 // use mm0 mask copy to merge a & b
4440 "pand %%mm0, %%mm2 \n\t"
4441 "pandn %%mm4, %%mm7 \n\t"
4442 "pandn %%mm1, %%mm0 \n\t"
4443 "paddw %%mm5, %%mm7 \n\t"
4444 "paddw %%mm2, %%mm0 \n\t"
4445 // test ((pa <= pb)? pa:pb) <= pc
4446 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4447 "pxor %%mm1, %%mm1 \n\t"
4448 "pand %%mm7, %%mm3 \n\t"
4449 "pandn %%mm0, %%mm7 \n\t"
4450 "paddw %%mm3, %%mm7 \n\t"
4451 "pxor %%mm0, %%mm0 \n\t"
4452 "packuswb %%mm1, %%mm7 \n\t"
4453 "movq (%0," PCX ",), %%mm3 \n\t" // load c=Prior(x-bpp)
4454 LOAD_GOT_rbp
4455 "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask)
4456 RESTORE_rbp
4457 "movq %%mm3, %%mm2 \n\t" // load b=Prior(x) step 1
4458 "paddb (%1," PCX ",), %%mm7 \n\t" // add Paeth predictor + Raw(x)
4459 "punpcklbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4460 "movq %%mm7, (%1," PCX ",) \n\t" // write back updated value
4461 "movq %%mm7, %%mm1 \n\t" // now mm1 will be used as
4462 // Raw(x-bpp)
4463 // do second set of 4 bytes
4464 "punpckhbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
4465 "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4466 // pav = p - a = (a + b - c) - a = b - c
4467 "movq %%mm2, %%mm4 \n\t"
4468 // pbv = p - b = (a + b - c) - b = a - c
4469 "movq %%mm1, %%mm5 \n\t"
4470 "psubw %%mm3, %%mm4 \n\t"
4471 "pxor %%mm7, %%mm7 \n\t"
4472 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4473 "movq %%mm4, %%mm6 \n\t"
4474 "psubw %%mm3, %%mm5 \n\t"
4475 // pa = abs(p-a) = abs(pav)
4476 // pb = abs(p-b) = abs(pbv)
4477 // pc = abs(p-c) = abs(pcv)
4478 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4479 "paddw %%mm5, %%mm6 \n\t"
4480 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4481 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4482 "psubw %%mm0, %%mm4 \n\t"
4483 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4484 "psubw %%mm0, %%mm4 \n\t"
4485 "psubw %%mm7, %%mm5 \n\t"
4486 "pxor %%mm0, %%mm0 \n\t"
4487 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4488 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4489 "psubw %%mm7, %%mm5 \n\t"
4490 "psubw %%mm0, %%mm6 \n\t"
4491 // test pa <= pb
4492 "movq %%mm4, %%mm7 \n\t"
4493 "psubw %%mm0, %%mm6 \n\t"
4494 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4495 "movq %%mm7, %%mm0 \n\t"
4496 // use mm7 mask to merge pa & pb
4497 "pand %%mm7, %%mm5 \n\t"
4498 // use mm0 mask copy to merge a & b
4499 "pand %%mm0, %%mm2 \n\t"
4500 "pandn %%mm4, %%mm7 \n\t"
4501 "pandn %%mm1, %%mm0 \n\t"
4502 "paddw %%mm5, %%mm7 \n\t"
4503 "paddw %%mm2, %%mm0 \n\t"
4504 // test ((pa <= pb)? pa:pb) <= pc
4505 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4506 "pxor %%mm1, %%mm1 \n\t"
4507 "pand %%mm7, %%mm3 \n\t"
4508 "pandn %%mm0, %%mm7 \n\t"
4509 "pxor %%mm1, %%mm1 \n\t"
4510 "paddw %%mm3, %%mm7 \n\t"
4511 "pxor %%mm0, %%mm0 \n\t"
4512 // step ecx to next set of 8 bytes and repeat loop til done
4513 "addl $8, %%ecx \n\t"
4514 "packuswb %%mm7, %%mm1 \n\t"
4515 "paddb -8(%1," PCX ",), %%mm1 \n\t" // add predictor with Raw(x)
4516 "cmpl %%eax, %%ecx \n\t" // MMXLength
4517 "movq %%mm1, -8(%1," PCX ",) \n\t" // write back updated value
4518 // mm1 will be used as Raw(x-bpp) next loop
4519 "jb paeth_4lp \n\t"
4520
4521 : "=S" (dummy_value_S), // output regs (dummy)
4522 "=D" (dummy_value_D),
4523 "=c" (dummy_value_c),
4524 "=a" (dummy_value_a)
4525
4526 : "0" (prev_row), // esi/rsi // input regs
4527 "1" (row), // edi/rdi
4528 "2" (diff), // ecx
4529 "3" (MMXLength) // eax
4530
4531 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
4532 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
4533 , "%mm4", "%mm5", "%mm6", "%mm7"
4534 #endif
4535 );
4536 }
4537 break; // end 4 bpp
4538
4539 case 1:
4540 case 2:
4541 {
4542 __asm__ __volatile__ (
4543 // preload "movl diff, %%eax \n\t" // eax: x = offset to align. bdry
4544 // preload "movl FullLength, %%edx \n\t"
4545 "cmpl %%edx, %%eax \n\t"
4546 "jnb paeth_dend \n\t"
4547
4548 SAVE_ebp
4549
4550 // preload "movl row, %2 \n\t" // edi/rdi
4551 // do Paeth decode for remaining bytes
4552 // preload "movl prev_row, %1 \n\t" // esi/rsi
4553 "movl %%eax, %%ebp \n\t"
4554 // preload "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx)
4555 "subl %%ecx, %%ebp \n\t" // ebp = eax - bpp
4556 "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx
4557
4558 SAVE_GOT_ebx
4559 SAVE_r11_r12_r13
4560
4561 "paeth_dlp: \n\t"
4562 "xorl %%ebx, %%ebx \n\t"
4563 // pav = p - a = (a + b - c) - a = b - c
4564 "movb (%1," PAX ",), %%bl \n\t" // load Prior(x) into bl
4565 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
4566 "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp)
4567 "movl %%ebx, " pa_TEMP " \n\t" // Save pav for later use
4568 "xorl %%ebx, %%ebx \n\t"
4569 // pbv = p - b = (a + b - c) - b = a - c
4570 "movb (%2," PBP ",), %%bl \n\t" // load Raw(x-bpp) into bl
4571 "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp)
4572 "movl %%ebx, %%ecx \n\t"
4573 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4574 "addl " pa_TEMP ", %%ebx \n\t" // pcv = pav + pbv
4575 // pc = abs(pcv)
4576 "testl $0x80000000, %%ebx \n\t"
4577 "jz paeth_dpca \n\t"
4578 "negl %%ebx \n\t" // reverse sign of neg values
4579
4580 "paeth_dpca: \n\t"
4581 "movl %%ebx, " pc_TEMP " \n\t" // save pc for later use
4582 // pb = abs(pbv)
4583 "testl $0x80000000, %%ecx \n\t"
4584 "jz paeth_dpba \n\t"
4585 "negl %%ecx \n\t" // reverse sign of neg values
4586
4587 "paeth_dpba: \n\t"
4588 "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use
4589 // pa = abs(pav)
4590 "movl " pa_TEMP ", %%ebx \n\t"
4591 "testl $0x80000000, %%ebx \n\t"
4592 "jz paeth_dpaa \n\t"
4593 "negl %%ebx \n\t" // reverse sign of neg values
4594
4595 "paeth_dpaa: \n\t"
4596 "movl %%ebx, " pa_TEMP " \n\t" // save pa for later use
4597 // test if pa <= pb
4598 "cmpl %%ecx, %%ebx \n\t"
4599 "jna paeth_dabb \n\t"
4600 // pa > pb; now test if pb <= pc
4601 "cmpl " pc_TEMP ", %%ecx \n\t"
4602 "jna paeth_dbbc \n\t"
4603 // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4604 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
4605 "jmp paeth_dpaeth \n\t"
4606
4607 "paeth_dbbc: \n\t"
4608 // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
4609 "movb (%1," PAX ",), %%cl \n\t" // load Prior(x) into cl
4610 "jmp paeth_dpaeth \n\t"
4611
4612 "paeth_dabb: \n\t"
4613 // pa <= pb; now test if pa <= pc
4614 "cmpl " pc_TEMP ", %%ebx \n\t"
4615 "jna paeth_dabc \n\t"
4616 // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
4617 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
4618 "jmp paeth_dpaeth \n\t"
4619
4620 "paeth_dabc: \n\t"
4621 // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
4622 "movb (%2," PBP ",), %%cl \n\t" // load Raw(x-bpp) into cl
4623
4624 "paeth_dpaeth: \n\t"
4625 "incl %%eax \n\t"
4626 "incl %%ebp \n\t"
4627 // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
4628 "addb %%cl, -1(%2," PAX ",) \n\t"
4629 "cmpl %%edx, %%eax \n\t" // check against FullLength
4630 "jb paeth_dlp \n\t"
4631
4632 RESTORE_r11_r12_r13
4633 RESTORE_GOT_ebx
4634 RESTORE_ebp
4635
4636 "paeth_dend: \n\t"
4637
4638 : "=c" (dummy_value_c), // output regs (dummy)
4639 "=S" (dummy_value_S),
4640 "=D" (dummy_value_D),
4641 "=a" (dummy_value_a),
4642 "=d" (dummy_value_d)
4643
4644 : "0" (bpp), // ecx // input regs
4645 "1" (prev_row), // esi/rsi
4646 "2" (row), // edi/rdi
4647 "3" (diff), // eax
4648 "4" (FullLength) // edx
4649
4650 CLOB_COLON_ebx_ebp_r1X // clobber list
4651 CLOBBER_GOT_ebx
4652 CLOB_COMMA_ebx_ebp
4653 CLOBBER_ebp
4654 CLOB_COMMA_ebX_r1X
4655 CLOBBER_r11_r12_r13
4656 );
4657 }
4658 return; // end 1 or 2 bpp (no need to go further with this one)
4659
4660 case 6:
4661 {
4662 // _ActiveMask2 = 0xffffffff00000000LL; // NOT USED ("_amask_0_4_4")
4663 // _ShiftBpp = 48; // bpp << 3 == bpp * 8
4664 // _ShiftRem = 16; // 64 - _ShiftBpp
4665
4666 __asm__ __volatile__ (
4667 // preload "movl diff, %%ecx \n\t"
4668 // preload "movl row, %1 \n\t" // edi/rdi
4669 // preload "movl prev_row, %0 \n\t" // esi/rsi
4670 // prime the pump: load the first Raw(x-bpp) data set
4671 "movq -8(%1," PCX ",), %%mm1 \n\t"
4672 "pxor %%mm0, %%mm0 \n\t"
4673
4674 "paeth_6lp: \n\t"
4675 // must shift to position Raw(x-bpp) data
4676 "psrlq $16, %%mm1 \n\t" // was _ShiftRem
4677 // do first set of 4 bytes
4678 "movq -8(%0," PCX ",), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4679 "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4680 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4681 "punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
4682 // must shift to position Prior(x-bpp) data
4683 "psrlq $16, %%mm3 \n\t" // was _ShiftRem
4684 // pav = p - a = (a + b - c) - a = b - c
4685 "movq %%mm2, %%mm4 \n\t"
4686 "punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c
4687 // pbv = p - b = (a + b - c) - b = a - c
4688 "movq %%mm1, %%mm5 \n\t"
4689 "psubw %%mm3, %%mm4 \n\t"
4690 "pxor %%mm7, %%mm7 \n\t"
4691 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4692 "movq %%mm4, %%mm6 \n\t"
4693 "psubw %%mm3, %%mm5 \n\t"
4694 // pa = abs(p-a) = abs(pav)
4695 // pb = abs(p-b) = abs(pbv)
4696 // pc = abs(p-c) = abs(pcv)
4697 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4698 "paddw %%mm5, %%mm6 \n\t"
4699 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4700 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4701 "psubw %%mm0, %%mm4 \n\t"
4702 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4703 "psubw %%mm0, %%mm4 \n\t"
4704 "psubw %%mm7, %%mm5 \n\t"
4705 "pxor %%mm0, %%mm0 \n\t"
4706 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4707 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4708 "psubw %%mm7, %%mm5 \n\t"
4709 "psubw %%mm0, %%mm6 \n\t"
4710 // test pa <= pb
4711 "movq %%mm4, %%mm7 \n\t"
4712 "psubw %%mm0, %%mm6 \n\t"
4713 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4714 "movq %%mm7, %%mm0 \n\t"
4715 // use mm7 mask to merge pa & pb
4716 "pand %%mm7, %%mm5 \n\t"
4717 // use mm0 mask copy to merge a & b
4718 "pand %%mm0, %%mm2 \n\t"
4719 "pandn %%mm4, %%mm7 \n\t"
4720 "pandn %%mm1, %%mm0 \n\t"
4721 "paddw %%mm5, %%mm7 \n\t"
4722 "paddw %%mm2, %%mm0 \n\t"
4723 // test ((pa <= pb)? pa:pb) <= pc
4724 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4725 "pxor %%mm1, %%mm1 \n\t"
4726 "pand %%mm7, %%mm3 \n\t"
4727 "pandn %%mm0, %%mm7 \n\t"
4728 "paddw %%mm3, %%mm7 \n\t"
4729 "pxor %%mm0, %%mm0 \n\t"
4730 "packuswb %%mm1, %%mm7 \n\t"
4731 "movq -8(%0," PCX ",), %%mm3 \n\t" // load c=Prior(x-bpp)
4732 LOAD_GOT_rbp
4733 "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask)
4734 RESTORE_rbp
4735 "psrlq $16, %%mm3 \n\t"
4736 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x) step 1
4737 "paddb (%1," PCX ",), %%mm7 \n\t" // add Paeth predictor + Raw(x)
4738 "movq %%mm2, %%mm6 \n\t"
4739 "movq %%mm7, (%1," PCX ",) \n\t" // write back updated value
4740 "movq -8(%1," PCX ",), %%mm1 \n\t"
4741 "psllq $48, %%mm6 \n\t" // bpp * 8 = bits per pixel
4742 "movq %%mm7, %%mm5 \n\t"
4743 "psrlq $16, %%mm1 \n\t" // 64 - (bpp * 8) = remainder
4744 "por %%mm6, %%mm3 \n\t"
4745 "psllq $48, %%mm5 \n\t" // was _ShiftBpp
4746 "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4747 "por %%mm5, %%mm1 \n\t"
4748 // do second set of 4 bytes
4749 "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4750 "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4751 // pav = p - a = (a + b - c) - a = b - c
4752 "movq %%mm2, %%mm4 \n\t"
4753 // pbv = p - b = (a + b - c) - b = a - c
4754 "movq %%mm1, %%mm5 \n\t"
4755 "psubw %%mm3, %%mm4 \n\t"
4756 "pxor %%mm7, %%mm7 \n\t"
4757 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4758 "movq %%mm4, %%mm6 \n\t"
4759 "psubw %%mm3, %%mm5 \n\t"
4760 // pa = abs(p-a) = abs(pav)
4761 // pb = abs(p-b) = abs(pbv)
4762 // pc = abs(p-c) = abs(pcv)
4763 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4764 "paddw %%mm5, %%mm6 \n\t"
4765 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4766 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4767 "psubw %%mm0, %%mm4 \n\t"
4768 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4769 "psubw %%mm0, %%mm4 \n\t"
4770 "psubw %%mm7, %%mm5 \n\t"
4771 "pxor %%mm0, %%mm0 \n\t"
4772 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4773 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4774 "psubw %%mm7, %%mm5 \n\t"
4775 "psubw %%mm0, %%mm6 \n\t"
4776 // test pa <= pb
4777 "movq %%mm4, %%mm7 \n\t"
4778 "psubw %%mm0, %%mm6 \n\t"
4779 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4780 "movq %%mm7, %%mm0 \n\t"
4781 // use mm7 mask to merge pa & pb
4782 "pand %%mm7, %%mm5 \n\t"
4783 // use mm0 mask copy to merge a & b
4784 "pand %%mm0, %%mm2 \n\t"
4785 "pandn %%mm4, %%mm7 \n\t"
4786 "pandn %%mm1, %%mm0 \n\t"
4787 "paddw %%mm5, %%mm7 \n\t"
4788 "paddw %%mm2, %%mm0 \n\t"
4789 // test ((pa <= pb)? pa:pb) <= pc
4790 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4791 "pxor %%mm1, %%mm1 \n\t"
4792 "pand %%mm7, %%mm3 \n\t"
4793 "pandn %%mm0, %%mm7 \n\t"
4794 "pxor %%mm1, %%mm1 \n\t"
4795 "paddw %%mm3, %%mm7 \n\t"
4796 "pxor %%mm0, %%mm0 \n\t"
4797 // step ecx to next set of 8 bytes and repeat loop til done
4798 "addl $8, %%ecx \n\t"
4799 "packuswb %%mm7, %%mm1 \n\t"
4800 "paddb -8(%1," PCX ",), %%mm1 \n\t" // add Paeth predictor + Raw(x)
4801 "cmpl %%eax, %%ecx \n\t" // MMXLength
4802 "movq %%mm1, -8(%1," PCX ",) \n\t" // write back updated value
4803 // mm1 will be used as Raw(x-bpp) next loop
4804 "jb paeth_6lp \n\t"
4805
4806 : "=S" (dummy_value_S), // output regs (dummy)
4807 "=D" (dummy_value_D),
4808 "=c" (dummy_value_c),
4809 "=a" (dummy_value_a)
4810
4811 : "0" (prev_row), // esi/rsi // input regs
4812 "1" (row), // edi/rdi
4813 "2" (diff), // ecx
4814 "3" (MMXLength) // eax
4815
4816 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
4817 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
4818 , "%mm4", "%mm5", "%mm6", "%mm7"
4819 #endif
4820 );
4821 }
4822 break; // end 6 bpp
4823
4824 case 8: // bpp == 8
4825 {
4826 __asm__ __volatile__ (
4827 // preload "movl diff, %%ecx \n\t"
4828 // preload "movl row, %1 \n\t" // edi/rdi
4829 // preload "movl prev_row, %0 \n\t" // esi/rsi
4830 "pxor %%mm0, %%mm0 \n\t"
4831 // prime the pump: load the first Raw(x-bpp) data set
4832 "movq -8(%1," PCX ",), %%mm1 \n\t" // only time should need to read
4833 // a=Raw(x-bpp) bytes
4834 "paeth_8lp: \n\t"
4835 // do first set of 4 bytes
4836 "movq -8(%0," PCX ",), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4837 "punpcklbw %%mm0, %%mm1 \n\t" // unpack Low bytes of a
4838 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4839 "punpcklbw %%mm0, %%mm2 \n\t" // unpack Low bytes of b
4840 // pav = p - a = (a + b - c) - a = b - c
4841 "movq %%mm2, %%mm4 \n\t"
4842 "punpcklbw %%mm0, %%mm3 \n\t" // unpack Low bytes of c
4843 // pbv = p - b = (a + b - c) - b = a - c
4844 "movq %%mm1, %%mm5 \n\t"
4845 "psubw %%mm3, %%mm4 \n\t"
4846 "pxor %%mm7, %%mm7 \n\t"
4847 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4848 "movq %%mm4, %%mm6 \n\t"
4849 "psubw %%mm3, %%mm5 \n\t"
4850 // pa = abs(p-a) = abs(pav)
4851 // pb = abs(p-b) = abs(pbv)
4852 // pc = abs(p-c) = abs(pcv)
4853 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4854 "paddw %%mm5, %%mm6 \n\t"
4855 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4856 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4857 "psubw %%mm0, %%mm4 \n\t"
4858 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4859 "psubw %%mm0, %%mm4 \n\t"
4860 "psubw %%mm7, %%mm5 \n\t"
4861 "pxor %%mm0, %%mm0 \n\t"
4862 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4863 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4864 "psubw %%mm7, %%mm5 \n\t"
4865 "psubw %%mm0, %%mm6 \n\t"
4866 // test pa <= pb
4867 "movq %%mm4, %%mm7 \n\t"
4868 "psubw %%mm0, %%mm6 \n\t"
4869 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4870 "movq %%mm7, %%mm0 \n\t"
4871 // use mm7 mask to merge pa & pb
4872 "pand %%mm7, %%mm5 \n\t"
4873 // use mm0 mask copy to merge a & b
4874 "pand %%mm0, %%mm2 \n\t"
4875 "pandn %%mm4, %%mm7 \n\t"
4876 "pandn %%mm1, %%mm0 \n\t"
4877 "paddw %%mm5, %%mm7 \n\t"
4878 "paddw %%mm2, %%mm0 \n\t"
4879 // test ((pa <= pb)? pa:pb) <= pc
4880 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4881 "pxor %%mm1, %%mm1 \n\t"
4882 "pand %%mm7, %%mm3 \n\t"
4883 "pandn %%mm0, %%mm7 \n\t"
4884 "paddw %%mm3, %%mm7 \n\t"
4885 "pxor %%mm0, %%mm0 \n\t"
4886 "packuswb %%mm1, %%mm7 \n\t"
4887 "movq -8(%0," PCX ",), %%mm3 \n\t" // read c=Prior(x-bpp) bytes
4888 LOAD_GOT_rbp
4889 "pand " AMASK4_4_0 ", %%mm7 \n\t" // _amask4_4_0 (was _ActiveMask)
4890 RESTORE_rbp
4891 "movq (%0," PCX ",), %%mm2 \n\t" // load b=Prior(x)
4892 "paddb (%1," PCX ",), %%mm7 \n\t" // add Paeth predictor + Raw(x)
4893 "punpckhbw %%mm0, %%mm3 \n\t" // unpack High bytes of c
4894 "movq %%mm7, (%1," PCX ",) \n\t" // write back updated value
4895 "movq -8(%1," PCX ",), %%mm1 \n\t" // read a=Raw(x-bpp) bytes
4896
4897 // do second set of 4 bytes
4898 "punpckhbw %%mm0, %%mm2 \n\t" // unpack High bytes of b
4899 "punpckhbw %%mm0, %%mm1 \n\t" // unpack High bytes of a
4900 // pav = p - a = (a + b - c) - a = b - c
4901 "movq %%mm2, %%mm4 \n\t"
4902 // pbv = p - b = (a + b - c) - b = a - c
4903 "movq %%mm1, %%mm5 \n\t"
4904 "psubw %%mm3, %%mm4 \n\t"
4905 "pxor %%mm7, %%mm7 \n\t"
4906 // pcv = p - c = (a + b - c) -c = (a - c) + (b - c) = pav + pbv
4907 "movq %%mm4, %%mm6 \n\t"
4908 "psubw %%mm3, %%mm5 \n\t"
4909 // pa = abs(p-a) = abs(pav)
4910 // pb = abs(p-b) = abs(pbv)
4911 // pc = abs(p-c) = abs(pcv)
4912 "pcmpgtw %%mm4, %%mm0 \n\t" // create mask pav bytes < 0
4913 "paddw %%mm5, %%mm6 \n\t"
4914 "pand %%mm4, %%mm0 \n\t" // only pav bytes < 0 in mm7
4915 "pcmpgtw %%mm5, %%mm7 \n\t" // create mask pbv bytes < 0
4916 "psubw %%mm0, %%mm4 \n\t"
4917 "pand %%mm5, %%mm7 \n\t" // only pbv bytes < 0 in mm0
4918 "psubw %%mm0, %%mm4 \n\t"
4919 "psubw %%mm7, %%mm5 \n\t"
4920 "pxor %%mm0, %%mm0 \n\t"
4921 "pcmpgtw %%mm6, %%mm0 \n\t" // create mask pcv bytes < 0
4922 "pand %%mm6, %%mm0 \n\t" // only pav bytes < 0 in mm7
4923 "psubw %%mm7, %%mm5 \n\t"
4924 "psubw %%mm0, %%mm6 \n\t"
4925 // test pa <= pb
4926 "movq %%mm4, %%mm7 \n\t"
4927 "psubw %%mm0, %%mm6 \n\t"
4928 "pcmpgtw %%mm5, %%mm7 \n\t" // pa > pb?
4929 "movq %%mm7, %%mm0 \n\t"
4930 // use mm7 mask to merge pa & pb
4931 "pand %%mm7, %%mm5 \n\t"
4932 // use mm0 mask copy to merge a & b
4933 "pand %%mm0, %%mm2 \n\t"
4934 "pandn %%mm4, %%mm7 \n\t"
4935 "pandn %%mm1, %%mm0 \n\t"
4936 "paddw %%mm5, %%mm7 \n\t"
4937 "paddw %%mm2, %%mm0 \n\t"
4938 // test ((pa <= pb)? pa:pb) <= pc
4939 "pcmpgtw %%mm6, %%mm7 \n\t" // pab > pc?
4940 "pxor %%mm1, %%mm1 \n\t"
4941 "pand %%mm7, %%mm3 \n\t"
4942 "pandn %%mm0, %%mm7 \n\t"
4943 "pxor %%mm1, %%mm1 \n\t"
4944 "paddw %%mm3, %%mm7 \n\t"
4945 "pxor %%mm0, %%mm0 \n\t"
4946 // step ecx to next set of 8 bytes and repeat loop til done
4947 "addl $8, %%ecx \n\t"
4948 "packuswb %%mm7, %%mm1 \n\t"
4949 "paddb -8(%1," PCX ",), %%mm1 \n\t" // add Paeth predictor + Raw(x)
4950 "cmpl %%eax, %%ecx \n\t" // MMXLength
4951 "movq %%mm1, -8(%1," PCX ",) \n\t" // write back updated value
4952 // mm1 will be used as Raw(x-bpp) next loop
4953 "jb paeth_8lp \n\t"
4954
4955 : "=S" (dummy_value_S), // output regs (dummy)
4956 "=D" (dummy_value_D),
4957 "=c" (dummy_value_c),
4958 "=a" (dummy_value_a)
4959
4960 : "0" (prev_row), // esi/rsi // input regs
4961 "1" (row), // edi/rdi
4962 "2" (diff), // ecx
4963 "3" (MMXLength) // eax
4964
4965 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
4966 : "%mm0", "%mm1", "%mm2", "%mm3" // clobber list
4967 , "%mm4", "%mm5", "%mm6", "%mm7"
4968 #endif
4969 );
4970 }
4971 break; // end 8 bpp
4972
4973 default: // bpp != 1,2,3,4,6,8: doesn't exist
4974 {
4975 // ERROR: SHOULD NEVER BE REACHED
4976 #if defined(PNG_DEBUG)
4977 png_debug(1, "Internal libpng logic error (GCC "
4978 "png_read_filter_row_mmx_paeth())\n");
4979 #endif
4980 }
4981 break;
4982
4983 } // end switch (bpp)
4984
4985 __asm__ __volatile__ (
4986 // MMX acceleration complete; now do clean-up
4987 // check if any remaining bytes left to decode
4988 //pre "movl FullLength, %%edx \n\t"
4989 //pre "movl MMXLength, %%eax \n\t"
4990 "cmpl %%edx, %%eax \n\t"
4991 "jnb paeth_end \n\t"
4992
4993 SAVE_ebp
4994
4995 //pre "movl row, %2 \n\t" // edi/rdi
4996 //pre "movl prev_row, %1 \n\t" // esi/rsi
4997 // do Paeth decode for remaining bytes
4998 "movl %%eax, %%ebp \n\t"
4999 //pre "subl bpp, %%ebp \n\t" // (bpp is preloaded into ecx)
5000 "subl %%ecx, %%ebp \n\t" // ebp = eax - bpp
5001 "xorl %%ecx, %%ecx \n\t" // zero ecx before using cl & cx below
5002
5003 SAVE_GOT_ebx
5004 SAVE_r11_r12_r13
5005
5006 "paeth_lp2: \n\t"
5007 "xorl %%ebx, %%ebx \n\t"
5008 // pav = p - a = (a + b - c) - a = b - c
5009 "movb (%1," PAX ",), %%bl \n\t" // load Prior(x) into bl
5010 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
5011 "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp)
5012 "movl %%ebx, " pa_TEMP " \n\t" // Save pav for later use
5013 "xorl %%ebx, %%ebx \n\t"
5014 // pbv = p - b = (a + b - c) - b = a - c
5015 "movb (%2," PBP ",), %%bl \n\t" // load Raw(x-bpp) into bl
5016 "subl %%ecx, %%ebx \n\t" // subtract Prior(x-bpp)
5017 "movl %%ebx, %%ecx \n\t"
5018 // pcv = p - c = (a + b - c) - c = (a - c) + (b - c) = pav + pbv
5019 "addl " pa_TEMP ", %%ebx \n\t" // pcv = pav + pbv
5020 // pc = abs(pcv)
5021 "testl $0x80000000, %%ebx \n\t"
5022 "jz paeth_pca2 \n\t"
5023 "negl %%ebx \n\t" // reverse sign of neg values
5024
5025 "paeth_pca2: \n\t"
5026 "movl %%ebx, " pc_TEMP " \n\t" // save pc for later use
5027 // pb = abs(pbv)
5028 "testl $0x80000000, %%ecx \n\t"
5029 "jz paeth_pba2 \n\t"
5030 "negl %%ecx \n\t" // reverse sign of neg values
5031
5032 "paeth_pba2: \n\t"
5033 "movl %%ecx, " pb_TEMP " \n\t" // save pb for later use
5034 // pa = abs(pav)
5035 "movl " pa_TEMP ", %%ebx \n\t"
5036 "testl $0x80000000, %%ebx \n\t"
5037 "jz paeth_paa2 \n\t"
5038 "negl %%ebx \n\t" // reverse sign of neg values
5039
5040 "paeth_paa2: \n\t"
5041 "movl %%ebx, " pa_TEMP " \n\t" // save pa for later use
5042 // test if pa <= pb
5043 "cmpl %%ecx, %%ebx \n\t"
5044 "jna paeth_abb2 \n\t"
5045 // pa > pb; now test if pb <= pc
5046 "cmpl " pc_TEMP ", %%ecx \n\t"
5047 "jna paeth_bbc2 \n\t"
5048 // pb > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
5049 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
5050 "jmp paeth_paeth2 \n\t"
5051
5052 "paeth_bbc2: \n\t"
5053 // pb <= pc; Raw(x) = Paeth(x) + Prior(x)
5054 "movb (%1," PAX ",), %%cl \n\t" // load Prior(x) into cl
5055 "jmp paeth_paeth2 \n\t"
5056
5057 "paeth_abb2: \n\t"
5058 // pa <= pb; now test if pa <= pc
5059 "cmpl " pc_TEMP ", %%ebx \n\t"
5060 "jna paeth_abc2 \n\t"
5061 // pa > pc; Raw(x) = Paeth(x) + Prior(x-bpp)
5062 "movb (%1," PBP ",), %%cl \n\t" // load Prior(x-bpp) into cl
5063 "jmp paeth_paeth2 \n\t"
5064
5065 "paeth_abc2: \n\t"
5066 // pa <= pc; Raw(x) = Paeth(x) + Raw(x-bpp)
5067 "movb (%2," PBP ",), %%cl \n\t" // load Raw(x-bpp) into cl
5068
5069 "paeth_paeth2: \n\t"
5070 "incl %%eax \n\t"
5071 "incl %%ebp \n\t"
5072 // Raw(x) = (Paeth(x) + Paeth_Predictor( a, b, c )) mod 256
5073 "addb %%cl, -1(%2," PAX ",) \n\t"
5074 "cmpl %%edx, %%eax \n\t" // check against FullLength
5075 "jb paeth_lp2 \n\t"
5076
5077 RESTORE_r11_r12_r13
5078 RESTORE_GOT_ebx
5079 RESTORE_ebp
5080
5081 "paeth_end: \n\t"
5082 "EMMS \n\t" // end MMX; prep for poss. FP instrs.
5083
5084 : "=c" (dummy_value_c), // output regs (dummy)
5085 "=S" (dummy_value_S),
5086 "=D" (dummy_value_D),
5087 "=a" (dummy_value_a),
5088 "=d" (dummy_value_d)
5089
5090 : "0" (bpp), // ecx // input regs
5091 "1" (prev_row), // esi/rsi
5092 "2" (row), // edi/rdi
5093 "3" (MMXLength), // eax
5094 "4" (FullLength) // edx
5095
5096 CLOB_COLON_ebx_ebp_r1X // clobber list
5097 CLOBBER_GOT_ebx
5098 CLOB_COMMA_ebx_ebp
5099 CLOBBER_ebp
5100 CLOB_COMMA_ebX_r1X
5101 CLOBBER_r11_r12_r13
5102 );
5103
5104 } /* end png_read_filter_row_mmx_paeth() */
5105
5106 #endif // PNG_x86_64_USE_GOTPCREL || PNG_THREAD_UNSAFE_OK
5107 #endif /* PNG_MMX_READ_FILTER_PAETH_SUPPORTED */
5108
5109
5110
5111
5112 #if defined(PNG_MMX_READ_FILTER_SUB_SUPPORTED)
5113
5114 //===========================================================================//
5115 // //
5116 // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ S U B //
5117 // //
5118 //===========================================================================//
5119
5120 // Optimized code for PNG Sub filter decoder
5121
5122 static void /* PRIVATE */
png_read_filter_row_mmx_sub(png_row_infop row_info,png_bytep row)5123 png_read_filter_row_mmx_sub(png_row_infop row_info, png_bytep row)
5124 {
5125 unsigned FullLength, MMXLength; // png_uint_32 is actually 64-bit on x86-64
5126 int bpp;
5127 int dummy_value_a;
5128 int dummy_value_c;
5129 int dummy_value_d;
5130 png_bytep dummy_value_D;
5131 int diff; // __attribute__((used));
5132
5133 bpp = (row_info->pixel_depth + 7) >> 3; // calc number of bytes per pixel
5134 FullLength = row_info->rowbytes - bpp; // number of bytes to filter
5135 // (why do we subtract off bpp? not so in avg or paeth...)
5136
5137 __asm__ __volatile__ (
5138 SAVE_r15
5139 SAVE_ebp
5140 //pre "movl row, %1 \n\t" // edi/rdi
5141 "mov %1, " PSI " \n\t" // lp = row
5142 //pre "movl bpp, %%ecx \n\t"
5143 "add " PCX ", %1 \n\t" // rp = row + bpp
5144 //pre "movl FullLength, %%eax \n\t" // bring in via eax...
5145 SAVE_FullLength // ...but store for later use
5146
5147 "xorl %%eax, %%eax \n\t"
5148
5149 // get # of bytes to alignment (note: computing _delta_ of two pointers,
5150 // so hereafter %%ebp is sufficient even on 64-bit)
5151 "mov %1, " PBP " \n\t" // take start of row
5152 "add $0xf, " PBP " \n\t" // add 7+8 to incr past alignment bdry
5153 // "andl $0xfffffff8, %%ebp \n\t" // mask to alignment boundary (32-bit!)
5154 CLEAR_BOTTOM_3_BITS PBP "\n\t" // mask to alignment boundary
5155 "sub %1, " PBP " \n\t" // subtract row ptr again => ebp =
5156 "jz sub_go \n\t" // target value of eax at alignment
5157
5158 "sub_lp1: \n\t" // fix alignment
5159 "movb (" PSI "," PAX ",), %%cl \n\t"
5160 "addb %%cl, (%1," PAX ",) \n\t"
5161 "incl %%eax \n\t"
5162 "cmpl %%ebp, %%eax \n\t"
5163 "jb sub_lp1 \n\t"
5164
5165 "sub_go: \n\t"
5166 RESTORE_FullLength "%%ecx \n\t" // FullLength -> ecx
5167 "movl %%ecx, %%edx \n\t"
5168 "subl %%eax, %%edx \n\t" // subtract alignment fix
5169 "andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8
5170 "subl %%edx, %%ecx \n\t" // drop over bytes from length
5171 //out "movl %%ecx, MMXLength \n\t"
5172 "movl %%ebp, %%eax \n\t" // ebp = diff, but no reg constraint(?)
5173 RESTORE_ebp // (could swap ebp and ecx functions,
5174 RESTORE_r15 // but %%cl issues...)
5175
5176 : "=c" (MMXLength), // 0 // output regs
5177 "=D" (dummy_value_D), // 1
5178 "=a" (diff) // 2
5179
5180 : "0" (bpp), // ecx // input regs
5181 "1" (row), // edi
5182 "2" (FullLength) // eax
5183
5184 : "%esi", "%edx" // clobber list
5185 _CLOBBER_r15
5186 _CLOBBER_ebp
5187 );
5188
5189 // now do the math for the rest of the row
5190 switch (bpp)
5191 {
5192 case 3:
5193 {
5194 // _ShiftBpp = 24; // == 3 * 8
5195 // _ShiftRem = 40; // == 64 - 24
5196
5197 __asm__ __volatile__ (
5198 // preload "mov row, %1 \n\t" // edi/rdi
5199 LOAD_GOT_rbp
5200 // load (former) _ActiveMask for 2nd active byte group
5201 "movq " AMASK2_3_3 ", %%mm7 \n\t" // _amask2_3_3
5202 RESTORE_rbp
5203
5204 // notused "mov %1, " PSI " \n\t" // lp = row
5205 // preload "movl bpp, %%ecx \n\t"
5206 "add " PCX ", %1 \n\t" // rp = row + bpp
5207 "movq %%mm7, %%mm6 \n\t"
5208 // preload "movl diff, %%edx \n\t"
5209 "psllq $24, %%mm6 \n\t" // move mask in mm6 to cover
5210 // 3rd active byte group
5211 // prime the pump: load the first Raw(x-bpp) data set
5212 "movq -8(%1," PDX ",), %%mm1 \n\t"
5213
5214 "sub_3lp: \n\t" // shift data for adding first
5215 "psrlq $40, %%mm1 \n\t" // bpp bytes (no need for mask;
5216 // shift clears inactive bytes)
5217 // add 1st active group
5218 "movq (%1," PDX ",), %%mm0 \n\t"
5219 "paddb %%mm1, %%mm0 \n\t"
5220
5221 // add 2nd active group
5222 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5223 "psllq $24, %%mm1 \n\t" // shift data to pos. correctly
5224 "pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group
5225 "paddb %%mm1, %%mm0 \n\t"
5226
5227 // add 3rd active group
5228 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5229 "psllq $24, %%mm1 \n\t" // shift data to pos. correctly
5230 "pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group
5231 "addl $8, %%edx \n\t"
5232 "paddb %%mm1, %%mm0 \n\t"
5233
5234 "cmpl %%eax, %%edx \n\t" // MMXLength
5235 "movq %%mm0, -8(%1," PDX ",) \n\t" // write updated Raws to array
5236 "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
5237 "jb sub_3lp \n\t"
5238
5239 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5240 "=D" (dummy_value_D), // 1
5241 "=d" (dummy_value_d), // 2
5242 "=a" (dummy_value_a) // 3
5243
5244 : "0" (bpp), // ecx // input regs
5245 "1" (row), // edi
5246 "2" (diff), // edx
5247 "3" (MMXLength) // eax
5248
5249 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
5250 : "%mm0", "%mm1", "%mm6", "%mm7" // clobber list
5251 #endif
5252 );
5253 }
5254 break; // end 3 bpp
5255
5256 case 4: // formerly shared with 6 bpp case via _ShiftBpp and _ShiftRem,
5257 { // but 64-bit PIC/.so problems (could still share, moving vars
5258 // into unused MMX regs via ecx/edx, but kludgy)
5259 // _ShiftBpp = bpp << 3; // 32 (psllq)
5260 // _ShiftRem = 64 - _ShiftBpp; // 32 (psrlq)
5261
5262 __asm__ __volatile__ (
5263 // preload "mov row, %1 \n\t" // edi/rdi
5264 // preload "movl diff, %%edx \n\t"
5265 // notused "mov %1, " PSI " \n\t" // lp = row
5266 // preload "movl bpp, %%ecx \n\t"
5267 "add " PCX ", %1 \n\t" // rp = row + bpp
5268
5269 // prime the pump: load the first Raw(x-bpp) data set
5270 "movq -8(%1," PDX ",), %%mm1 \n\t"
5271
5272 "sub_4lp: \n\t" // shift data for adding first
5273 "psrlq $32, %%mm1 \n\t" // bpp bytes (no need for mask;
5274 // shift clears inactive bytes)
5275 "movq (%1," PDX ",), %%mm0 \n\t"
5276 "paddb %%mm1, %%mm0 \n\t"
5277
5278 // add 2nd active group
5279 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5280 "psllq $32, %%mm1 \n\t" // shift data to pos. correctly
5281 "addl $8, %%edx \n\t"
5282 "paddb %%mm1, %%mm0 \n\t"
5283
5284 "cmpl %%eax, %%edx \n\t" // MMXLength
5285 "movq %%mm0, -8(%1," PDX ",) \n\t" // write updated Raws to array
5286 "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
5287 "jb sub_4lp \n\t"
5288
5289 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5290 "=D" (dummy_value_D), // 1
5291 "=d" (dummy_value_d), // 2
5292 "=a" (dummy_value_a) // 3
5293
5294 : "0" (bpp), // ecx // input regs
5295 "1" (row), // edi
5296 "2" (diff), // edx
5297 "3" (MMXLength) // eax
5298
5299 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
5300 : "%mm0", "%mm1" // clobber list
5301 #endif
5302 );
5303 }
5304 break; // end 4 bpp
5305
5306 case 1:
5307 {
5308 __asm__ __volatile__ (
5309 // preload "movl diff, %%edx \n\t"
5310 // preload "mov row, %1 \n\t" // edi/rdi
5311 // preload "cmpl FullLength, %%edx \n\t"
5312 "cmpl %%eax, %%edx \n\t"
5313 "jnb sub_1end \n\t"
5314 "mov %1, " PSI " \n\t" // lp = row
5315 // irrel. "xorl %%ecx, %%ecx \n\t" // (actually bug with preload)
5316 // preload "movl bpp, %%ecx \n\t"
5317 "add " PCX ", %1 \n\t" // rp = row + bpp
5318
5319 "sub_1lp: \n\t"
5320 "movb (" PSI "," PDX ",), %%cl \n\t"
5321 "addb %%cl, (%1," PDX ",) \n\t"
5322 "incl %%edx \n\t"
5323 "cmpl %%eax, %%edx \n\t" // compare with FullLength
5324 "jb sub_1lp \n\t"
5325
5326 "sub_1end: \n\t"
5327
5328 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5329 "=D" (dummy_value_D), // 1
5330 "=d" (dummy_value_d), // 2
5331 "=a" (dummy_value_a) // 3
5332
5333 : "0" (bpp), // ecx // input regs
5334 "1" (row), // edi
5335 "2" (diff), // edx
5336 "3" (FullLength) // eax
5337
5338 : "%esi" // clobber list
5339 );
5340 }
5341 return; // end 1 bpp (bypassing cleanup block!)
5342
5343 case 2:
5344 {
5345 // _ShiftBpp = 16; // == 2 * 8
5346 // _ShiftRem = 48; // == 64 - 16
5347
5348 __asm__ __volatile__ (
5349 LOAD_GOT_rbp
5350 // load (former) _ActiveMask for 2nd active byte group
5351 "movq " AMASK4_2_2 ", %%mm7 \n\t" // _amask4_2_2
5352 RESTORE_rbp
5353 // preload "movl diff, %%edx \n\t"
5354 "movq %%mm7, %%mm6 \n\t"
5355 // preload "mov row, %1 \n\t" // edi/rdi
5356 "psllq $16, %%mm6 \n\t" // move mask in mm6 to cover
5357 // 3rd active byte group
5358 // notused "mov %1, " PSI " \n\t" // lp = row
5359 "movq %%mm6, %%mm5 \n\t"
5360 // preload "movl bpp, %%ecx \n\t"
5361 "add " PCX ", %1 \n\t" // rp = row + bpp
5362 "psllq $16, %%mm5 \n\t" // move mask in mm5 to cover
5363 // 4th active byte group
5364 // prime the pump: load the first Raw(x-bpp) data set
5365 "movq -8(%1," PDX ",), %%mm1 \n\t"
5366
5367 "sub_2lp: \n\t" // shift data for adding first
5368 "psrlq $48, %%mm1 \n\t" // bpp bytes (no need for mask;
5369 // shift clears inactive bytes)
5370 // add 1st active group
5371 "movq (%1," PDX ",), %%mm0 \n\t"
5372 "paddb %%mm1, %%mm0 \n\t"
5373
5374 // add 2nd active group
5375 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5376 "psllq $16, %%mm1 \n\t" // shift data to pos. correctly
5377 "pand %%mm7, %%mm1 \n\t" // mask to use 2nd active group
5378 "paddb %%mm1, %%mm0 \n\t"
5379
5380 // add 3rd active group
5381 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5382 "psllq $16, %%mm1 \n\t" // shift data to pos. correctly
5383 "pand %%mm6, %%mm1 \n\t" // mask to use 3rd active group
5384 "paddb %%mm1, %%mm0 \n\t"
5385
5386 // add 4th active group
5387 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5388 "psllq $16, %%mm1 \n\t" // shift data to pos. correctly
5389 "pand %%mm5, %%mm1 \n\t" // mask to use 4th active group
5390 "addl $8, %%edx \n\t"
5391 "paddb %%mm1, %%mm0 \n\t"
5392 "cmpl %%eax, %%edx \n\t" // MMXLength
5393 "movq %%mm0, -8(%1," PDX ",) \n\t" // write updated Raws to array
5394 "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
5395 "jb sub_2lp \n\t"
5396
5397 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5398 "=D" (dummy_value_D), // 1
5399 "=d" (dummy_value_d), // 2
5400 "=a" (dummy_value_a) // 3
5401
5402 : "0" (bpp), // ecx // input regs
5403 "1" (row), // edi
5404 "2" (diff), // edx
5405 "3" (MMXLength) // eax
5406
5407 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
5408 : "%mm0", "%mm1", "%mm5", "%mm6" // clobber list
5409 , "%mm7"
5410 #endif
5411 );
5412 }
5413 break; // end 2 bpp
5414
5415 case 6: // formerly shared with 4 bpp case (see comments there)
5416 {
5417 // _ShiftBpp = bpp << 3; // 48 (psllq)
5418 // _ShiftRem = 64 - _ShiftBpp; // 16 (psrlq)
5419
5420 __asm__ __volatile__ (
5421 // preload "mov row, %1 \n\t" // edi/rdi
5422 // preload "movl diff, %%edx \n\t"
5423 // notused "mov %1, " PSI " \n\t" // lp = row
5424 // preload "movl bpp, %%ecx \n\t"
5425 "add " PCX ", %1 \n\t" // rp = row + bpp
5426
5427 // prime the pump: load the first Raw(x-bpp) data set
5428 "movq -8(%1," PDX ",), %%mm1 \n\t"
5429
5430 "sub_6lp: \n\t" // shift data for adding first
5431 "psrlq $16, %%mm1 \n\t" // bpp bytes (no need for mask;
5432 // shift clears inactive bytes)
5433 "movq (%1," PDX ",), %%mm0 \n\t"
5434 "paddb %%mm1, %%mm0 \n\t"
5435
5436 // add 2nd active group
5437 "movq %%mm0, %%mm1 \n\t" // mov updated Raws to mm1
5438 "psllq $48, %%mm1 \n\t" // shift data to pos. correctly
5439 "addl $8, %%edx \n\t"
5440 "paddb %%mm1, %%mm0 \n\t"
5441
5442 "cmpl %%eax, %%edx \n\t" // MMXLength
5443 "movq %%mm0, -8(%1," PDX ",) \n\t" // write updated Raws to array
5444 "movq %%mm0, %%mm1 \n\t" // prep 1st add at top of loop
5445 "jb sub_6lp \n\t"
5446
5447 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5448 "=D" (dummy_value_D), // 1
5449 "=d" (dummy_value_d), // 2
5450 "=a" (dummy_value_a) // 3
5451
5452 : "0" (bpp), // ecx // input regs
5453 "1" (row), // edi
5454 "2" (diff), // edx
5455 "3" (MMXLength) // eax
5456
5457 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
5458 : "%mm0", "%mm1" // clobber list
5459 #endif
5460 );
5461 }
5462 break; // end 6 bpp
5463
5464 case 8:
5465 {
5466 __asm__ __volatile__ (
5467 // preload "mov row, %1 \n\t" // edi/rdi
5468 // preload "movl diff, %%edx \n\t"
5469 // notused "mov %1, " PSI " \n\t" // lp = row
5470 // preload "movl bpp, %%ecx \n\t"
5471 "add " PCX ", %1 \n\t" // rp = row + bpp
5472 // preload "movl MMXLength, %%eax \n\t"
5473
5474 // prime the pump: load the first Raw(x-bpp) data set
5475 "movq -8(%1," PDX ",), %%mm7 \n\t"
5476 "movl %%eax, %%esi \n\t" // copy of MMXLength -> esi
5477 "andl $0x0000003f, %%esi \n\t" // calc bytes over mult of 64
5478
5479 "sub_8lp: \n\t"
5480 "movq (%1," PDX ",), %%mm0 \n\t" // load Sub(x) for 1st 8 bytes
5481 "paddb %%mm7, %%mm0 \n\t"
5482 "movq 8(%1," PDX ",), %%mm1 \n\t" // load Sub(x) for 2nd 8 bytes
5483 "movq %%mm0, (%1," PDX ",) \n\t" // write Raw(x) for 1st 8 bytes
5484
5485 // Now mm0 will be used as Raw(x-bpp) for the 2nd group of 8 bytes.
5486 // This will be repeated for each group of 8 bytes with the 8th
5487 // group being used as the Raw(x-bpp) for the 1st group of the
5488 // next loop.
5489
5490 "paddb %%mm0, %%mm1 \n\t"
5491 "movq 16(%1," PDX ",), %%mm2 \n\t" // load Sub(x) for 3rd 8 bytes
5492 "movq %%mm1, 8(%1," PDX ",) \n\t" // write Raw(x) for 2nd 8 bytes
5493 "paddb %%mm1, %%mm2 \n\t"
5494 "movq 24(%1," PDX ",), %%mm3 \n\t" // load Sub(x) for 4th 8 bytes
5495 "movq %%mm2, 16(%1," PDX ",) \n\t" // write Raw(x) for 3rd 8 bytes
5496 "paddb %%mm2, %%mm3 \n\t"
5497 "movq 32(%1," PDX ",), %%mm4 \n\t" // load Sub(x) for 5th 8 bytes
5498 "movq %%mm3, 24(%1," PDX ",) \n\t" // write Raw(x) for 4th 8 bytes
5499 "paddb %%mm3, %%mm4 \n\t"
5500 "movq 40(%1," PDX ",), %%mm5 \n\t" // load Sub(x) for 6th 8 bytes
5501 "movq %%mm4, 32(%1," PDX ",) \n\t" // write Raw(x) for 5th 8 bytes
5502 "paddb %%mm4, %%mm5 \n\t"
5503 "movq 48(%1," PDX ",), %%mm6 \n\t" // load Sub(x) for 7th 8 bytes
5504 "movq %%mm5, 40(%1," PDX ",) \n\t" // write Raw(x) for 6th 8 bytes
5505 "paddb %%mm5, %%mm6 \n\t"
5506 "movq 56(%1," PDX ",), %%mm7 \n\t" // load Sub(x) for 8th 8 bytes
5507 "movq %%mm6, 48(%1," PDX ",) \n\t" // write Raw(x) for 7th 8 bytes
5508 "addl $64, %%edx \n\t"
5509 "paddb %%mm6, %%mm7 \n\t"
5510 "cmpl %%esi, %%edx \n\t" // cmp to bytes over mult of 64
5511 "movq %%mm7, -8(%1," PDX ",) \n\t" // write Raw(x) for 8th 8 bytes
5512 "jb sub_8lp \n\t"
5513
5514 "cmpl %%eax, %%edx \n\t" // compare to MMXLength
5515 "jnb sub_8lt8 \n\t"
5516
5517 "sub_8lpA: \n\t"
5518 "movq (%1," PDX ",), %%mm0 \n\t"
5519 "addl $8, %%edx \n\t"
5520 "paddb %%mm7, %%mm0 \n\t"
5521 "cmpl %%eax, %%edx \n\t" // compare to MMXLength
5522 "movq %%mm0, -8(%1," PDX ",) \n\t" // -8 to offset early addl edx
5523 "movq %%mm0, %%mm7 \n\t" // move calculated Raw(x) data
5524 "jb sub_8lpA \n\t" // to mm7 to be new Raw(x-bpp)
5525 // for next loop
5526 "sub_8lt8: \n\t"
5527
5528 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5529 "=D" (dummy_value_D), // 1
5530 "=d" (dummy_value_d), // 2
5531 "=a" (dummy_value_a) // 3
5532
5533 : "0" (bpp), // ecx // input regs
5534 "1" (row), // edi
5535 "2" (diff), // edx
5536 "3" (MMXLength) // eax
5537
5538 : "%esi" // clobber list
5539 #if defined(CLOBBER_MMX_REGS_SUPPORTED)
5540 , "%mm0", "%mm1", "%mm2", "%mm3"
5541 , "%mm4", "%mm5", "%mm6", "%mm7"
5542 #endif
5543 );
5544 }
5545 break; // end 8 bpp
5546
5547 default: // bpp != 1,2,3,4,6,8: doesn't exist
5548 {
5549 // ERROR: SHOULD NEVER BE REACHED
5550 #if defined(PNG_DEBUG)
5551 png_debug(1, "Internal libpng logic error (GCC "
5552 "png_read_filter_row_mmx_sub())\n");
5553 #endif
5554 }
5555 break;
5556
5557 } // end switch (bpp)
5558
5559 __asm__ __volatile__ (
5560 //pre "movl MMXLength, %%eax \n\t"
5561 //pre "mov row, %1 \n\t" // edi/rdi
5562 //pre "cmpl FullLength, %%eax \n\t"
5563 "cmpl %%edx, %%eax \n\t"
5564 "jnb sub_end \n\t"
5565
5566 "mov %1, " PSI " \n\t" // lp = row
5567 //pre "movl bpp, %%ecx \n\t"
5568 "add " PCX ", %1 \n\t" // rp = row + bpp
5569 "xorl %%ecx, %%ecx \n\t"
5570
5571 "sub_lp2: \n\t"
5572 "movb (" PSI "," PAX ",), %%cl \n\t"
5573 "addb %%cl, (%1," PAX ",) \n\t"
5574 "incl %%eax \n\t"
5575 "cmpl %%edx, %%eax \n\t" // FullLength
5576 "jb sub_lp2 \n\t"
5577
5578 "sub_end: \n\t"
5579 "EMMS \n\t" // end MMX instructions
5580
5581 : "=c" (dummy_value_c), // 0 // output regs (dummy)
5582 "=D" (dummy_value_D), // 1
5583 "=a" (dummy_value_a), // 2
5584 "=d" (dummy_value_d) // 3
5585
5586 : "0" (bpp), // ecx // input regs
5587 "1" (row), // edi
5588 "2" (MMXLength), // eax
5589 "3" (FullLength) // edx
5590
5591 : "%esi" // clobber list
5592 );
5593
5594 } // end of png_read_filter_row_mmx_sub()
5595
5596 #endif /* PNG_MMX_READ_FILTER_SUB_SUPPORTED */
5597
5598
5599
5600
5601 #if defined(PNG_MMX_READ_FILTER_UP_SUPPORTED)
5602
5603 //===========================================================================//
5604 // //
5605 // P N G _ R E A D _ F I L T E R _ R O W _ M M X _ U P //
5606 // //
5607 //===========================================================================//
5608
5609 // Optimized code for PNG Up filter decoder
5610
5611 static void /* PRIVATE */
png_read_filter_row_mmx_up(png_row_infop row_info,png_bytep row,png_bytep prev_row)5612 png_read_filter_row_mmx_up(png_row_infop row_info, png_bytep row,
5613 png_bytep prev_row)
5614 {
5615 unsigned len; // png_uint_32 is actually 64-bit on x86-64
5616 int dummy_value_d; // fix 'forbidden register 3 (dx) was spilled' error
5617 png_bytep dummy_value_S;
5618 png_bytep dummy_value_D;
5619
5620 len = row_info->rowbytes; // number of bytes to filter
5621
5622 __asm__ __volatile__ (
5623 SAVE_GOT_ebx
5624 //pre "mov prev_row, %1 \n\t" // esi/rsi
5625 //pre "movl row, %2 \n\t" // edi/rdi
5626
5627 "xorl %%ebx, %%ebx \n\t"
5628 "xorl %%eax, %%eax \n\t"
5629
5630 // get # of bytes to alignment (note: computing _delta_ of two pointers,
5631 // so hereafter %%ecx is sufficient even on 64-bit)
5632 "mov %2, " PCX " \n\t" // take start of row
5633 "add $0x7, " PCX " \n\t" // add 7 to incr past alignment bdry
5634 // "andl $0xfffffff8, %%ecx \n\t" // mask to alignment boundary (32-bit!)
5635 CLEAR_BOTTOM_3_BITS PCX "\n\t" // mask to alignment boundary
5636 "sub %2, " PCX " \n\t" // subtract row ptr again => ebp =
5637 "jz up_go \n\t" // target value of ecx at alignment
5638
5639 "up_lp1: \n\t" // fix alignment
5640 "movb (%2," PBX ",), %%al \n\t"
5641 "addb (%1," PBX ",), %%al \n\t"
5642 "incl %%ebx \n\t"
5643 "cmpl %%ecx, %%ebx \n\t"
5644 "movb %%al, -1(%2," PBX ",) \n\t" // mov does not affect flags; -1 to
5645 "jb up_lp1 \n\t" // offset incl ebx
5646
5647 "up_go: \n\t"
5648 //pre "movl len, %%edx \n\t"
5649 "movl %%edx, %%ecx \n\t"
5650 "subl %%ebx, %%edx \n\t" // subtract alignment fix
5651 "andl $0x0000003f, %%edx \n\t" // calc bytes over mult of 64
5652 "subl %%edx, %%ecx \n\t" // sub over-bytes from original length
5653
5654 // unrolled loop - use all MMX registers and interleave to reduce
5655 // number of branch instructions (loops) and reduce partial stalls
5656 "up_loop: \n\t"
5657 "movq (%1," PBX ",), %%mm1 \n\t"
5658 "movq (%2," PBX ",), %%mm0 \n\t"
5659 "movq 8(%1," PBX ",), %%mm3 \n\t"
5660 "paddb %%mm1, %%mm0 \n\t"
5661 "movq 8(%2," PBX ",), %%mm2 \n\t"
5662 "movq %%mm0, (%2," PBX ",) \n\t"
5663 "paddb %%mm3, %%mm2 \n\t"
5664 "movq 16(%1," PBX ",), %%mm5 \n\t"
5665 "movq %%mm2, 8(%2," PBX ",) \n\t"
5666 "movq 16(%2," PBX ",), %%mm4 \n\t"
5667 "movq 24(%1," PBX ",), %%mm7 \n\t"
5668 "paddb %%mm5, %%mm4 \n\t"
5669 "movq 24(%2," PBX ",), %%mm6 \n\t"
5670 "movq %%mm4, 16(%2," PBX ",) \n\t"
5671 "paddb %%mm7, %%mm6 \n\t"
5672 "movq 32(%1," PBX ",), %%mm1 \n\t"
5673 "movq %%mm6, 24(%2," PBX ",) \n\t"
5674 "movq 32(%2," PBX ",), %%mm0 \n\t"
5675 "movq 40(%1," PBX ",), %%mm3 \n\t"
5676 "paddb %%mm1, %%mm0 \n\t"
5677 "movq 40(%2," PBX ",), %%mm2 \n\t"
5678 "movq %%mm0, 32(%2," PBX ",) \n\t"
5679 "paddb %%mm3, %%mm2 \n\t"
5680 "movq 48(%1," PBX ",), %%mm5 \n\t"
5681 "movq %%mm2, 40(%2," PBX ",) \n\t"
5682 "movq 48(%2," PBX ",), %%mm4 \n\t"
5683 "movq 56(%1," PBX ",), %%mm7 \n\t"
5684 "paddb %%mm5, %%mm4 \n\t"
5685 "movq 56(%2," PBX ",), %%mm6 \n\t"
5686 "movq %%mm4, 48(%2," PBX ",) \n\t"
5687 "addl $64, %%ebx \n\t"
5688 "paddb %%mm7, %%mm6 \n\t"
5689 "cmpl %%ecx, %%ebx \n\t"
5690 "movq %%mm6, -8(%2," PBX ",) \n\t" // (+56)movq does not affect flags;
5691 "jb up_loop \n\t" // -8 to offset addl ebx
5692
5693 "cmpl $0, %%edx \n\t" // test for bytes over mult of 64
5694 "jz up_end \n\t"
5695
5696 "cmpl $8, %%edx \n\t" // test for less than 8 bytes
5697 "jb up_lt8 \n\t" // [added by lcreeve at netins.net]
5698
5699 "addl %%edx, %%ecx \n\t"
5700 "andl $0x00000007, %%edx \n\t" // calc bytes over mult of 8
5701 "subl %%edx, %%ecx \n\t" // drop over-bytes from length
5702 "jz up_lt8 \n\t"
5703
5704 "up_lpA: \n\t" // use MMX regs to update 8 bytes sim.
5705 "movq (%1," PBX ",), %%mm1 \n\t"
5706 "movq (%2," PBX ",), %%mm0 \n\t"
5707 "addl $8, %%ebx \n\t"
5708 "paddb %%mm1, %%mm0 \n\t"
5709 "cmpl %%ecx, %%ebx \n\t"
5710 "movq %%mm0, -8(%2," PBX ",) \n\t" // movq does not affect flags; -8 to
5711 "jb up_lpA \n\t" // offset add ebx
5712 "cmpl $0, %%edx \n\t" // test for bytes over mult of 8
5713 "jz up_end \n\t"
5714
5715 "up_lt8: \n\t"
5716 "xorl %%eax, %%eax \n\t"
5717 "addl %%edx, %%ecx \n\t" // move over byte count into counter
5718
5719 "up_lp2: \n\t" // use x86 regs for remaining bytes
5720 "movb (%2," PBX ",), %%al \n\t"
5721 "addb (%1," PBX ",), %%al \n\t"
5722 "incl %%ebx \n\t"
5723 "cmpl %%ecx, %%ebx \n\t"
5724 "movb %%al, -1(%2," PBX ",) \n\t" // mov does not affect flags; -1 to
5725 "jb up_lp2 \n\t" // offset inc ebx
5726
5727 "up_end: \n\t"
5728 "EMMS \n\t" // conversion of filtered row complete
5729 RESTORE_GOT_ebx
5730
5731 : "=d" (dummy_value_d), // 0 // output regs (dummy)
5732 "=S" (dummy_value_S), // 1
5733 "=D" (dummy_value_D) // 2
5734
5735 : "0" (len), // edx // input regs
5736 "1" (prev_row), // esi
5737 "2" (row) // edi
5738
5739 : "%eax", "%ecx" // clobber list (no input regs!)
5740 _CLOBBER_GOT_ebx
5741 #if defined(PNG_CLOBBER_MMX_REGS_SUPPORTED)
5742 , "%mm0", "%mm1", "%mm2", "%mm3"
5743 , "%mm4", "%mm5", "%mm6", "%mm7"
5744 #endif
5745 );
5746
5747 } // end of png_read_filter_row_mmx_up()
5748
5749 #endif /* PNG_MMX_READ_FILTER_UP_SUPPORTED */
5750
5751
5752
5753
5754 /*===========================================================================*/
5755 /* */
5756 /* P N G _ R E A D _ F I L T E R _ R O W */
5757 /* */
5758 /*===========================================================================*/
5759
5760 /* Optimized png_read_filter_row routines */
5761
5762 void /* PRIVATE */
png_read_filter_row(png_structp png_ptr,png_row_infop row_info,png_bytep row,png_bytep prev_row,int filter)5763 png_read_filter_row(png_structp png_ptr, png_row_infop row_info, png_bytep
5764 row, png_bytep prev_row, int filter)
5765 {
5766 #if defined(PNG_DEBUG)
5767 char filtname[10];
5768 #endif
5769
5770 if (_mmx_supported == 2) {
5771 #if !defined(PNG_1_0_X)
5772 /* this should have happened in png_init_mmx_flags() already */
5773 png_warning(png_ptr, "asm_flags may not have been initialized");
5774 #endif
5775 png_mmx_support();
5776 }
5777
5778 #if defined(PNG_DEBUG)
5779 png_debug(1, "in png_read_filter_row (pnggccrd.c)\n");
5780 switch (filter)
5781 {
5782 case 0:
5783 png_snprintf(filtname, 10, "none");
5784 break;
5785
5786 case 1:
5787 png_snprintf(filtname, 10, "sub-%s",
5788 #ifdef PNG_MMX_READ_FILTER_SUB_SUPPORTED
5789 #if !defined(PNG_1_0_X)
5790 ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
5791 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5792 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5793 #else
5794 _mmx_supported
5795 #endif
5796 ? "MMX" :
5797 #endif
5798 "C");
5799 break;
5800
5801 case 2:
5802 png_snprintf(filtname, 10, "up-%s",
5803 #ifdef PNG_MMX_READ_FILTER_UP_SUPPORTED
5804 #if !defined(PNG_1_0_X)
5805 ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
5806 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5807 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5808 #else
5809 _mmx_supported
5810 #endif
5811 ? "MMX" :
5812 #endif
5813 "C");
5814 break;
5815
5816 case 3:
5817 png_snprintf(filtname, 10, "avg-%s",
5818 #ifdef PNG_MMX_READ_FILTER_AVG_SUPPORTED
5819 #if !defined(PNG_1_0_X)
5820 ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
5821 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5822 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5823 #else
5824 _mmx_supported
5825 #endif
5826 ? "MMX" :
5827 #endif
5828 "C");
5829 break;
5830
5831 case 4:
5832 png_snprintf(filtname, 10, "paeth-%s",
5833 #ifdef PNG_MMX_READ_FILTER_PAETH_SUPPORTED
5834 #if defined(PNG_x86_64_USE_GOTPCREL) || defined(PNG_THREAD_UNSAFE_OK)
5835 #if !defined(PNG_1_0_X)
5836 ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
5837 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5838 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5839 #else
5840 _mmx_supported
5841 #endif
5842 ? "MMX" :
5843 #endif /* PNG_x86_64_USE_GOTPCREL || PNG_THREAD_UNSAFE_OK */
5844 #endif
5845 "C");
5846 break;
5847
5848 default:
5849 png_snprintf(filtname, 10, "unknown");
5850 break;
5851 }
5852 png_debug2(2, "row_number=%ld, %s, ", png_ptr->row_number, filtname);
5853 //png_debug1(0, "png_ptr=%10p, ", png_ptr);
5854 //png_debug1(0, "asm_flags=0x%08lx, ", png_ptr->asm_flags);
5855 png_debug1(0, "row=%10p, ", row);
5856 png_debug2(0, "pixdepth=%d, bytes=%d, ", (int)row_info->pixel_depth,
5857 (int)((row_info->pixel_depth + 7) >> 3));
5858 png_debug1(0, "rowbytes=%ld\n", row_info->rowbytes);
5859 #endif /* PNG_DEBUG */
5860
5861 switch (filter)
5862 {
5863 case PNG_FILTER_VALUE_NONE:
5864 break;
5865
5866 case PNG_FILTER_VALUE_SUB:
5867 #ifdef PNG_MMX_READ_FILTER_SUB_SUPPORTED
5868 #if !defined(PNG_1_0_X)
5869 if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_SUB) &&
5870 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5871 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5872 #else
5873 if (_mmx_supported)
5874 #endif
5875 {
5876 png_read_filter_row_mmx_sub(row_info, row);
5877 }
5878 else
5879 #endif
5880 {
5881 png_uint_32 i;
5882 png_uint_32 istop = row_info->rowbytes;
5883 png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5884 png_bytep rp = row + bpp;
5885 png_bytep lp = row;
5886
5887 for (i = bpp; i < istop; i++)
5888 {
5889 *rp = (png_byte)(((int)(*rp) + (int)(*lp++)) & 0xff);
5890 rp++;
5891 }
5892 } /* end !UseMMX_sub */
5893 break;
5894
5895 case PNG_FILTER_VALUE_UP:
5896 #ifdef PNG_MMX_READ_FILTER_UP_SUPPORTED
5897 #if !defined(PNG_1_0_X)
5898 if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_UP) &&
5899 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5900 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5901 #else
5902 if (_mmx_supported)
5903 #endif
5904 {
5905 png_read_filter_row_mmx_up(row_info, row, prev_row);
5906 }
5907 else
5908 #endif
5909 {
5910 png_uint_32 i;
5911 png_uint_32 istop = row_info->rowbytes;
5912 png_bytep rp = row;
5913 png_bytep pp = prev_row;
5914
5915 for (i = 0; i < istop; ++i)
5916 {
5917 *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5918 rp++;
5919 }
5920 } /* end !UseMMX_up */
5921 break;
5922
5923 case PNG_FILTER_VALUE_AVG:
5924 #ifdef PNG_MMX_READ_FILTER_AVG_SUPPORTED
5925 #if !defined(PNG_1_0_X)
5926 if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_AVG) &&
5927 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5928 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5929 #else
5930 if (_mmx_supported)
5931 #endif
5932 {
5933 png_read_filter_row_mmx_avg(row_info, row, prev_row);
5934 }
5935 else
5936 #endif
5937 {
5938 png_uint_32 i;
5939 png_bytep rp = row;
5940 png_bytep pp = prev_row;
5941 png_bytep lp = row;
5942 png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5943 png_uint_32 istop = row_info->rowbytes - bpp;
5944
5945 for (i = 0; i < bpp; i++)
5946 {
5947 *rp = (png_byte)(((int)(*rp) +
5948 ((int)(*pp++) >> 1)) & 0xff);
5949 rp++;
5950 }
5951
5952 for (i = 0; i < istop; i++)
5953 {
5954 *rp = (png_byte)(((int)(*rp) +
5955 ((int)(*pp++ + *lp++) >> 1)) & 0xff);
5956 rp++;
5957 }
5958 } /* end !UseMMX_avg */
5959 break;
5960
5961 case PNG_FILTER_VALUE_PAETH:
5962 #ifdef PNG_MMX_READ_FILTER_PAETH_SUPPORTED
5963 #if defined(PNG_x86_64_USE_GOTPCREL) || defined(PNG_THREAD_UNSAFE_OK)
5964 #if !defined(PNG_1_0_X)
5965 if ((png_ptr->asm_flags & PNG_ASM_FLAG_MMX_READ_FILTER_PAETH) &&
5966 (row_info->pixel_depth >= png_ptr->mmx_bitdepth_threshold) &&
5967 (row_info->rowbytes >= png_ptr->mmx_rowbytes_threshold))
5968 #else
5969 if (_mmx_supported)
5970 #endif
5971 {
5972 png_read_filter_row_mmx_paeth(row_info, row, prev_row);
5973 }
5974 else
5975 #endif /* PNG_x86_64_USE_GOTPCREL || PNG_THREAD_UNSAFE_OK */
5976 #endif
5977 {
5978 png_uint_32 i;
5979 png_bytep rp = row;
5980 png_bytep pp = prev_row;
5981 png_bytep lp = row;
5982 png_bytep cp = prev_row;
5983 png_uint_32 bpp = (row_info->pixel_depth + 7) >> 3;
5984 png_uint_32 istop = row_info->rowbytes - bpp;
5985
5986 for (i = 0; i < bpp; i++)
5987 {
5988 *rp = (png_byte)(((int)(*rp) + (int)(*pp++)) & 0xff);
5989 rp++;
5990 }
5991
5992 for (i = 0; i < istop; i++) /* use leftover rp,pp */
5993 {
5994 int a, b, c, pa, pb, pc, p;
5995
5996 a = *lp++;
5997 b = *pp++;
5998 c = *cp++;
5999
6000 p = b - c;
6001 pc = a - c;
6002
6003 #if defined(PNG_USE_ABS)
6004 pa = abs(p);
6005 pb = abs(pc);
6006 pc = abs(p + pc);
6007 #else
6008 pa = p < 0 ? -p : p;
6009 pb = pc < 0 ? -pc : pc;
6010 pc = (p + pc) < 0 ? -(p + pc) : p + pc;
6011 #endif
6012
6013 /*
6014 if (pa <= pb && pa <= pc)
6015 p = a;
6016 else if (pb <= pc)
6017 p = b;
6018 else
6019 p = c;
6020 */
6021
6022 p = (pa <= pb && pa <= pc) ? a : (pb <= pc) ? b : c;
6023
6024 *rp = (png_byte)(((int)(*rp) + p) & 0xff);
6025 rp++;
6026 }
6027 } /* end !UseMMX_paeth */
6028 break;
6029
6030 default:
6031 png_warning(png_ptr, "Ignoring bad row-filter type");
6032 *row=0;
6033 break;
6034 }
6035 }
6036
6037 #endif /* PNG_HAVE_MMX_READ_FILTER_ROW */
6038
6039
6040 #endif /* PNG_ASSEMBLER_CODE_SUPPORTED && PNG_USE_PNGGCCRD */
6041 #endif /* __GNUC__ */
6042