1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // Speed-critical functions. 11 // 12 // Author: Skal (pascal.massimino@gmail.com) 13 14 #ifndef WEBP_DSP_DSP_H_ 15 #define WEBP_DSP_DSP_H_ 16 17 #include "../webp/types.h" 18 #include "../utils/utils.h" 19 20 #ifdef __cplusplus 21 extern "C" { 22 #endif 23 24 #define BPS 32 // this is the common stride for enc/dec 25 26 //------------------------------------------------------------------------------ 27 // CPU detection 28 29 #if defined(__GNUC__) 30 # define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) 31 # define LOCAL_GCC_PREREQ(maj, min) \ 32 (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) 33 #else 34 # define LOCAL_GCC_VERSION 0 35 # define LOCAL_GCC_PREREQ(maj, min) 0 36 #endif 37 38 #ifndef __has_builtin 39 # define __has_builtin(x) 0 40 #endif 41 42 #if defined(_MSC_VER) && _MSC_VER > 1310 && \ 43 (defined(_M_X64) || defined(_M_IX86)) 44 #define WEBP_MSC_SSE2 // Visual C++ SSE2 targets 45 #endif 46 47 #if defined(_MSC_VER) && _MSC_VER >= 1500 && \ 48 (defined(_M_X64) || defined(_M_IX86)) 49 #define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets 50 #endif 51 52 // WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp 53 // files without intrinsics, allowing the corresponding Init() to be called. 54 // Files containing intrinsics will need to be built targeting the instruction 55 // set so should succeed on one of the earlier tests. 56 #if defined(__SSE2__) || defined(WEBP_MSC_SSE2) || defined(WEBP_HAVE_SSE2) 57 #define WEBP_USE_SSE2 58 #endif 59 60 #if defined(__SSE4_1__) || defined(WEBP_MSC_SSE41) || defined(WEBP_HAVE_SSE41) 61 #define WEBP_USE_SSE41 62 #endif 63 64 #if defined(__AVX2__) || defined(WEBP_HAVE_AVX2) 65 #define WEBP_USE_AVX2 66 #endif 67 68 #if defined(__ANDROID__) && defined(__ARM_ARCH_7A__) 69 #define WEBP_ANDROID_NEON // Android targets that might support NEON 70 #endif 71 72 // The intrinsics currently cause compiler errors with arm-nacl-gcc and the 73 // inline assembly would need to be modified for use with Native Client. 74 #if (defined(__ARM_NEON__) || defined(__aarch64__)) \ 75 && !defined(__native_client__) 76 #define WEBP_USE_NEON 77 #endif 78 79 #if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) 80 #define WEBP_USE_NEON 81 #define WEBP_USE_INTRINSICS 82 #endif 83 84 #if defined(__mips__) && !defined(__mips64) && \ 85 defined(__mips_isa_rev) && (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) 86 #define WEBP_USE_MIPS32 87 #if (__mips_isa_rev >= 2) 88 #define WEBP_USE_MIPS32_R2 89 #if defined(__mips_dspr2) || (__mips_dsp_rev >= 2) 90 #define WEBP_USE_MIPS_DSP_R2 91 #endif 92 #endif 93 #endif 94 95 // This macro prevents thread_sanitizer from reporting known concurrent writes. 96 #define WEBP_TSAN_IGNORE_FUNCTION 97 #if defined(__has_feature) 98 #if __has_feature(thread_sanitizer) 99 #undef WEBP_TSAN_IGNORE_FUNCTION 100 #define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) 101 #endif 102 #endif 103 104 typedef enum { 105 kSSE2, 106 kSSE3, 107 kSSE4_1, 108 kAVX, 109 kAVX2, 110 kNEON, 111 kMIPS32, 112 kMIPSdspR2 113 } CPUFeature; 114 // returns true if the CPU supports the feature. 115 typedef int (*VP8CPUInfo)(CPUFeature feature); 116 WEBP_EXTERN(VP8CPUInfo) VP8GetCPUInfo; 117 118 //------------------------------------------------------------------------------ 119 // Init stub generator 120 121 // Defines an init function stub to ensure each module exposes a symbol, 122 // avoiding a compiler warning. 123 #define WEBP_DSP_INIT_STUB(func) \ 124 extern void func(void); \ 125 WEBP_TSAN_IGNORE_FUNCTION void func(void) {} 126 127 //------------------------------------------------------------------------------ 128 // Encoding 129 130 // Transforms 131 // VP8Idct: Does one of two inverse transforms. If do_two is set, the transforms 132 // will be done for (ref, in, dst) and (ref + 4, in + 16, dst + 4). 133 typedef void (*VP8Idct)(const uint8_t* ref, const int16_t* in, uint8_t* dst, 134 int do_two); 135 typedef void (*VP8Fdct)(const uint8_t* src, const uint8_t* ref, int16_t* out); 136 typedef void (*VP8WHT)(const int16_t* in, int16_t* out); 137 extern VP8Idct VP8ITransform; 138 extern VP8Fdct VP8FTransform; 139 extern VP8Fdct VP8FTransform2; // performs two transforms at a time 140 extern VP8WHT VP8FTransformWHT; 141 // Predictions 142 // *dst is the destination block. *top and *left can be NULL. 143 typedef void (*VP8IntraPreds)(uint8_t *dst, const uint8_t* left, 144 const uint8_t* top); 145 typedef void (*VP8Intra4Preds)(uint8_t *dst, const uint8_t* top); 146 extern VP8Intra4Preds VP8EncPredLuma4; 147 extern VP8IntraPreds VP8EncPredLuma16; 148 extern VP8IntraPreds VP8EncPredChroma8; 149 150 typedef int (*VP8Metric)(const uint8_t* pix, const uint8_t* ref); 151 extern VP8Metric VP8SSE16x16, VP8SSE16x8, VP8SSE8x8, VP8SSE4x4; 152 typedef int (*VP8WMetric)(const uint8_t* pix, const uint8_t* ref, 153 const uint16_t* const weights); 154 extern VP8WMetric VP8TDisto4x4, VP8TDisto16x16; 155 156 typedef void (*VP8BlockCopy)(const uint8_t* src, uint8_t* dst); 157 extern VP8BlockCopy VP8Copy4x4; 158 extern VP8BlockCopy VP8Copy16x8; 159 // Quantization 160 struct VP8Matrix; // forward declaration 161 typedef int (*VP8QuantizeBlock)(int16_t in[16], int16_t out[16], 162 const struct VP8Matrix* const mtx); 163 // Same as VP8QuantizeBlock, but quantizes two consecutive blocks. 164 typedef int (*VP8Quantize2Blocks)(int16_t in[32], int16_t out[32], 165 const struct VP8Matrix* const mtx); 166 167 extern VP8QuantizeBlock VP8EncQuantizeBlock; 168 extern VP8Quantize2Blocks VP8EncQuantize2Blocks; 169 170 // specific to 2nd transform: 171 typedef int (*VP8QuantizeBlockWHT)(int16_t in[16], int16_t out[16], 172 const struct VP8Matrix* const mtx); 173 extern VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; 174 175 extern const int VP8DspScan[16 + 4 + 4]; 176 177 // Collect histogram for susceptibility calculation. 178 #define MAX_COEFF_THRESH 31 // size of histogram used by CollectHistogram. 179 typedef struct { 180 // We only need to store max_value and last_non_zero, not the distribution. 181 int max_value; 182 int last_non_zero; 183 } VP8Histogram; 184 typedef void (*VP8CHisto)(const uint8_t* ref, const uint8_t* pred, 185 int start_block, int end_block, 186 VP8Histogram* const histo); 187 extern VP8CHisto VP8CollectHistogram; 188 // General-purpose util function to help VP8CollectHistogram(). 189 void VP8SetHistogramData(const int distribution[MAX_COEFF_THRESH + 1], 190 VP8Histogram* const histo); 191 192 // must be called before using any of the above 193 void VP8EncDspInit(void); 194 195 //------------------------------------------------------------------------------ 196 // cost functions (encoding) 197 198 extern const uint16_t VP8EntropyCost[256]; // 8bit fixed-point log(p) 199 // approximate cost per level: 200 extern const uint16_t VP8LevelFixedCosts[2047 /*MAX_LEVEL*/ + 1]; 201 extern const uint8_t VP8EncBands[16 + 1]; 202 203 struct VP8Residual; 204 typedef void (*VP8SetResidualCoeffsFunc)(const int16_t* const coeffs, 205 struct VP8Residual* const res); 206 extern VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; 207 208 // Cost calculation function. 209 typedef int (*VP8GetResidualCostFunc)(int ctx0, 210 const struct VP8Residual* const res); 211 extern VP8GetResidualCostFunc VP8GetResidualCost; 212 213 // must be called before anything using the above 214 void VP8EncDspCostInit(void); 215 216 //------------------------------------------------------------------------------ 217 // Decoding 218 219 typedef void (*VP8DecIdct)(const int16_t* coeffs, uint8_t* dst); 220 // when doing two transforms, coeffs is actually int16_t[2][16]. 221 typedef void (*VP8DecIdct2)(const int16_t* coeffs, uint8_t* dst, int do_two); 222 extern VP8DecIdct2 VP8Transform; 223 extern VP8DecIdct VP8TransformAC3; 224 extern VP8DecIdct VP8TransformUV; 225 extern VP8DecIdct VP8TransformDC; 226 extern VP8DecIdct VP8TransformDCUV; 227 extern VP8WHT VP8TransformWHT; 228 229 // *dst is the destination block, with stride BPS. Boundary samples are 230 // assumed accessible when needed. 231 typedef void (*VP8PredFunc)(uint8_t* dst); 232 extern VP8PredFunc VP8PredLuma16[/* NUM_B_DC_MODES */]; 233 extern VP8PredFunc VP8PredChroma8[/* NUM_B_DC_MODES */]; 234 extern VP8PredFunc VP8PredLuma4[/* NUM_BMODES */]; 235 236 // clipping tables (for filtering) 237 extern const int8_t* const VP8ksclip1; // clips [-1020, 1020] to [-128, 127] 238 extern const int8_t* const VP8ksclip2; // clips [-112, 112] to [-16, 15] 239 extern const uint8_t* const VP8kclip1; // clips [-255,511] to [0,255] 240 extern const uint8_t* const VP8kabs0; // abs(x) for x in [-255,255] 241 // must be called first 242 void VP8InitClipTables(void); 243 244 // simple filter (only for luma) 245 typedef void (*VP8SimpleFilterFunc)(uint8_t* p, int stride, int thresh); 246 extern VP8SimpleFilterFunc VP8SimpleVFilter16; 247 extern VP8SimpleFilterFunc VP8SimpleHFilter16; 248 extern VP8SimpleFilterFunc VP8SimpleVFilter16i; // filter 3 inner edges 249 extern VP8SimpleFilterFunc VP8SimpleHFilter16i; 250 251 // regular filter (on both macroblock edges and inner edges) 252 typedef void (*VP8LumaFilterFunc)(uint8_t* luma, int stride, 253 int thresh, int ithresh, int hev_t); 254 typedef void (*VP8ChromaFilterFunc)(uint8_t* u, uint8_t* v, int stride, 255 int thresh, int ithresh, int hev_t); 256 // on outer edge 257 extern VP8LumaFilterFunc VP8VFilter16; 258 extern VP8LumaFilterFunc VP8HFilter16; 259 extern VP8ChromaFilterFunc VP8VFilter8; 260 extern VP8ChromaFilterFunc VP8HFilter8; 261 262 // on inner edge 263 extern VP8LumaFilterFunc VP8VFilter16i; // filtering 3 inner edges altogether 264 extern VP8LumaFilterFunc VP8HFilter16i; 265 extern VP8ChromaFilterFunc VP8VFilter8i; // filtering u and v altogether 266 extern VP8ChromaFilterFunc VP8HFilter8i; 267 268 // must be called before anything using the above 269 void VP8DspInit(void); 270 271 //------------------------------------------------------------------------------ 272 // WebP I/O 273 274 #define FANCY_UPSAMPLING // undefined to remove fancy upsampling support 275 276 // Convert a pair of y/u/v lines together to the output rgb/a colorspace. 277 // bottom_y can be NULL if only one line of output is needed (at top/bottom). 278 typedef void (*WebPUpsampleLinePairFunc)( 279 const uint8_t* top_y, const uint8_t* bottom_y, 280 const uint8_t* top_u, const uint8_t* top_v, 281 const uint8_t* cur_u, const uint8_t* cur_v, 282 uint8_t* top_dst, uint8_t* bottom_dst, int len); 283 284 #ifdef FANCY_UPSAMPLING 285 286 // Fancy upsampling functions to convert YUV to RGB(A) modes 287 extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */]; 288 289 #endif // FANCY_UPSAMPLING 290 291 // Per-row point-sampling methods. 292 typedef void (*WebPSamplerRowFunc)(const uint8_t* y, 293 const uint8_t* u, const uint8_t* v, 294 uint8_t* dst, int len); 295 // Generic function to apply 'WebPSamplerRowFunc' to the whole plane: 296 void WebPSamplerProcessPlane(const uint8_t* y, int y_stride, 297 const uint8_t* u, const uint8_t* v, int uv_stride, 298 uint8_t* dst, int dst_stride, 299 int width, int height, WebPSamplerRowFunc func); 300 301 // Sampling functions to convert rows of YUV to RGB(A) 302 extern WebPSamplerRowFunc WebPSamplers[/* MODE_LAST */]; 303 304 // General function for converting two lines of ARGB or RGBA. 305 // 'alpha_is_last' should be true if 0xff000000 is stored in memory as 306 // as 0x00, 0x00, 0x00, 0xff (little endian). 307 WebPUpsampleLinePairFunc WebPGetLinePairConverter(int alpha_is_last); 308 309 // YUV444->RGB converters 310 typedef void (*WebPYUV444Converter)(const uint8_t* y, 311 const uint8_t* u, const uint8_t* v, 312 uint8_t* dst, int len); 313 314 extern WebPYUV444Converter WebPYUV444Converters[/* MODE_LAST */]; 315 316 // Must be called before using the WebPUpsamplers[] (and for premultiplied 317 // colorspaces like rgbA, rgbA4444, etc) 318 void WebPInitUpsamplers(void); 319 // Must be called before using WebPSamplers[] 320 void WebPInitSamplers(void); 321 // Must be called before using WebPYUV444Converters[] 322 void WebPInitYUV444Converters(void); 323 324 //------------------------------------------------------------------------------ 325 // ARGB -> YUV converters 326 327 // Convert ARGB samples to luma Y. 328 extern void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); 329 // Convert ARGB samples to U/V with downsampling. do_store should be '1' for 330 // even lines and '0' for odd ones. 'src_width' is the original width, not 331 // the U/V one. 332 extern void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, 333 int src_width, int do_store); 334 335 // Convert a row of accumulated (four-values) of rgba32 toward U/V 336 extern void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, 337 uint8_t* u, uint8_t* v, int width); 338 339 // Convert RGB or BGR to Y 340 extern void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); 341 extern void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); 342 343 // used for plain-C fallback. 344 extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, 345 int src_width, int do_store); 346 extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, 347 uint8_t* u, uint8_t* v, int width); 348 349 // Must be called before using the above. 350 void WebPInitConvertARGBToYUV(void); 351 352 //------------------------------------------------------------------------------ 353 // Rescaler 354 355 struct WebPRescaler; 356 357 // Import a row of data and save its contribution in the rescaler. 358 // 'channel' denotes the channel number to be imported. 'Expand' corresponds to 359 // the wrk->x_expand case. Otherwise, 'Shrink' is to be used. 360 typedef void (*WebPRescalerImportRowFunc)(struct WebPRescaler* const wrk, 361 const uint8_t* src); 362 363 extern WebPRescalerImportRowFunc WebPRescalerImportRowExpand; 364 extern WebPRescalerImportRowFunc WebPRescalerImportRowShrink; 365 366 // Export one row (starting at x_out position) from rescaler. 367 // 'Expand' corresponds to the wrk->y_expand case. 368 // Otherwise 'Shrink' is to be used 369 typedef void (*WebPRescalerExportRowFunc)(struct WebPRescaler* const wrk); 370 extern WebPRescalerExportRowFunc WebPRescalerExportRowExpand; 371 extern WebPRescalerExportRowFunc WebPRescalerExportRowShrink; 372 373 // Plain-C implementation, as fall-back. 374 extern void WebPRescalerImportRowExpandC(struct WebPRescaler* const wrk, 375 const uint8_t* src); 376 extern void WebPRescalerImportRowShrinkC(struct WebPRescaler* const wrk, 377 const uint8_t* src); 378 extern void WebPRescalerExportRowExpandC(struct WebPRescaler* const wrk); 379 extern void WebPRescalerExportRowShrinkC(struct WebPRescaler* const wrk); 380 381 // Main entry calls: 382 extern void WebPRescalerImportRow(struct WebPRescaler* const wrk, 383 const uint8_t* src); 384 // Export one row (starting at x_out position) from rescaler. 385 extern void WebPRescalerExportRow(struct WebPRescaler* const wrk); 386 387 // Must be called first before using the above. 388 void WebPRescalerDspInit(void); 389 390 //------------------------------------------------------------------------------ 391 // Utilities for processing transparent channel. 392 393 // Apply alpha pre-multiply on an rgba, bgra or argb plane of size w * h. 394 // alpha_first should be 0 for argb, 1 for rgba or bgra (where alpha is last). 395 extern void (*WebPApplyAlphaMultiply)( 396 uint8_t* rgba, int alpha_first, int w, int h, int stride); 397 398 // Same, buf specifically for RGBA4444 format 399 extern void (*WebPApplyAlphaMultiply4444)( 400 uint8_t* rgba4444, int w, int h, int stride); 401 402 // Dispatch the values from alpha[] plane to the ARGB destination 'dst'. 403 // Returns true if alpha[] plane has non-trivial values different from 0xff. 404 extern int (*WebPDispatchAlpha)(const uint8_t* alpha, int alpha_stride, 405 int width, int height, 406 uint8_t* dst, int dst_stride); 407 408 // Transfer packed 8b alpha[] values to green channel in dst[], zero'ing the 409 // A/R/B values. 'dst_stride' is the stride for dst[] in uint32_t units. 410 extern void (*WebPDispatchAlphaToGreen)(const uint8_t* alpha, int alpha_stride, 411 int width, int height, 412 uint32_t* dst, int dst_stride); 413 414 // Extract the alpha values from 32b values in argb[] and pack them into alpha[] 415 // (this is the opposite of WebPDispatchAlpha). 416 // Returns true if there's only trivial 0xff alpha values. 417 extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride, 418 int width, int height, 419 uint8_t* alpha, int alpha_stride); 420 421 // Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B). 422 // Un-Multiply operation transforms x into x * 255 / A. 423 424 // Pre-Multiply or Un-Multiply (if 'inverse' is true) argb values in a row. 425 extern void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse); 426 427 // Same a WebPMultARGBRow(), but for several rows. 428 void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, 429 int inverse); 430 431 // Same for a row of single values, with side alpha values. 432 extern void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, 433 int width, int inverse); 434 435 // Same a WebPMultRow(), but for several 'num_rows' rows. 436 void WebPMultRows(uint8_t* ptr, int stride, 437 const uint8_t* alpha, int alpha_stride, 438 int width, int num_rows, int inverse); 439 440 // Plain-C versions, used as fallback by some implementations. 441 void WebPMultRowC(uint8_t* const ptr, const uint8_t* const alpha, 442 int width, int inverse); 443 void WebPMultARGBRowC(uint32_t* const ptr, int width, int inverse); 444 445 // To be called first before using the above. 446 void WebPInitAlphaProcessing(void); 447 448 // ARGB packing function: a/r/g/b input is rgba or bgra order. 449 extern void (*VP8PackARGB)(const uint8_t* a, const uint8_t* r, 450 const uint8_t* g, const uint8_t* b, int len, 451 uint32_t* out); 452 453 // RGB packing function. 'step' can be 3 or 4. r/g/b input is rgb or bgr order. 454 extern void (*VP8PackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, 455 int len, int step, uint32_t* out); 456 457 // To be called first before using the above. 458 void VP8EncDspARGBInit(void); 459 460 //------------------------------------------------------------------------------ 461 // Filter functions 462 463 typedef enum { // Filter types. 464 WEBP_FILTER_NONE = 0, 465 WEBP_FILTER_HORIZONTAL, 466 WEBP_FILTER_VERTICAL, 467 WEBP_FILTER_GRADIENT, 468 WEBP_FILTER_LAST = WEBP_FILTER_GRADIENT + 1, // end marker 469 WEBP_FILTER_BEST, // meta-types 470 WEBP_FILTER_FAST 471 } WEBP_FILTER_TYPE; 472 473 typedef void (*WebPFilterFunc)(const uint8_t* in, int width, int height, 474 int stride, uint8_t* out); 475 typedef void (*WebPUnfilterFunc)(int width, int height, int stride, 476 int row, int num_rows, uint8_t* data); 477 478 // Filter the given data using the given predictor. 479 // 'in' corresponds to a 2-dimensional pixel array of size (stride * height) 480 // in raster order. 481 // 'stride' is number of bytes per scan line (with possible padding). 482 // 'out' should be pre-allocated. 483 extern WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; 484 485 // In-place reconstruct the original data from the given filtered data. 486 // The reconstruction will be done for 'num_rows' rows starting from 'row' 487 // (assuming rows upto 'row - 1' are already reconstructed). 488 extern WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; 489 490 // To be called first before using the above. 491 void VP8FiltersInit(void); 492 493 #ifdef __cplusplus 494 } // extern "C" 495 #endif 496 497 #endif /* WEBP_DSP_DSP_H_ */ 498