Lines Matching refs:__m128i
30 const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); in CollectHistogramSSE2()
37 const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); in CollectHistogramSSE2()
38 const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); in CollectHistogramSSE2()
40 const __m128i sign0 = _mm_srai_epi16(out0, 15); in CollectHistogramSSE2()
41 const __m128i sign1 = _mm_srai_epi16(out1, 15); in CollectHistogramSSE2()
43 const __m128i xor0 = _mm_xor_si128(out0, sign0); in CollectHistogramSSE2()
44 const __m128i xor1 = _mm_xor_si128(out1, sign1); in CollectHistogramSSE2()
45 const __m128i abs0 = _mm_sub_epi16(xor0, sign0); in CollectHistogramSSE2()
46 const __m128i abs1 = _mm_sub_epi16(xor1, sign1); in CollectHistogramSSE2()
48 const __m128i v0 = _mm_srai_epi16(abs0, 2); in CollectHistogramSSE2()
49 const __m128i v1 = _mm_srai_epi16(abs1, 2); in CollectHistogramSSE2()
51 const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); in CollectHistogramSSE2()
52 const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); in CollectHistogramSSE2()
54 _mm_storeu_si128((__m128i*)&out[0], bin0); in CollectHistogramSSE2()
55 _mm_storeu_si128((__m128i*)&out[8], bin1); in CollectHistogramSSE2()
89 const __m128i k1 = _mm_set1_epi16(20091); in ITransformSSE2()
90 const __m128i k2 = _mm_set1_epi16(-30068); in ITransformSSE2()
91 __m128i T0, T1, T2, T3; in ITransformSSE2()
97 __m128i in0, in1, in2, in3; in ITransformSSE2()
99 in0 = _mm_loadl_epi64((__m128i*)&in[0]); in ITransformSSE2()
100 in1 = _mm_loadl_epi64((__m128i*)&in[4]); in ITransformSSE2()
101 in2 = _mm_loadl_epi64((__m128i*)&in[8]); in ITransformSSE2()
102 in3 = _mm_loadl_epi64((__m128i*)&in[12]); in ITransformSSE2()
108 const __m128i inB0 = _mm_loadl_epi64((__m128i*)&in[16]); in ITransformSSE2()
109 const __m128i inB1 = _mm_loadl_epi64((__m128i*)&in[20]); in ITransformSSE2()
110 const __m128i inB2 = _mm_loadl_epi64((__m128i*)&in[24]); in ITransformSSE2()
111 const __m128i inB3 = _mm_loadl_epi64((__m128i*)&in[28]); in ITransformSSE2()
127 const __m128i a = _mm_add_epi16(in0, in2); in ITransformSSE2()
128 const __m128i b = _mm_sub_epi16(in0, in2); in ITransformSSE2()
130 const __m128i c1 = _mm_mulhi_epi16(in1, k2); in ITransformSSE2()
131 const __m128i c2 = _mm_mulhi_epi16(in3, k1); in ITransformSSE2()
132 const __m128i c3 = _mm_sub_epi16(in1, in3); in ITransformSSE2()
133 const __m128i c4 = _mm_sub_epi16(c1, c2); in ITransformSSE2()
134 const __m128i c = _mm_add_epi16(c3, c4); in ITransformSSE2()
136 const __m128i d1 = _mm_mulhi_epi16(in1, k1); in ITransformSSE2()
137 const __m128i d2 = _mm_mulhi_epi16(in3, k2); in ITransformSSE2()
138 const __m128i d3 = _mm_add_epi16(in1, in3); in ITransformSSE2()
139 const __m128i d4 = _mm_add_epi16(d1, d2); in ITransformSSE2()
140 const __m128i d = _mm_add_epi16(d3, d4); in ITransformSSE2()
143 const __m128i tmp0 = _mm_add_epi16(a, d); in ITransformSSE2()
144 const __m128i tmp1 = _mm_add_epi16(b, c); in ITransformSSE2()
145 const __m128i tmp2 = _mm_sub_epi16(b, c); in ITransformSSE2()
146 const __m128i tmp3 = _mm_sub_epi16(a, d); in ITransformSSE2()
153 const __m128i transpose0_0 = _mm_unpacklo_epi16(tmp0, tmp1); in ITransformSSE2()
154 const __m128i transpose0_1 = _mm_unpacklo_epi16(tmp2, tmp3); in ITransformSSE2()
155 const __m128i transpose0_2 = _mm_unpackhi_epi16(tmp0, tmp1); in ITransformSSE2()
156 const __m128i transpose0_3 = _mm_unpackhi_epi16(tmp2, tmp3); in ITransformSSE2()
161 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); in ITransformSSE2()
162 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); in ITransformSSE2()
163 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); in ITransformSSE2()
164 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); in ITransformSSE2()
183 const __m128i four = _mm_set1_epi16(4); in ITransformSSE2()
184 const __m128i dc = _mm_add_epi16(T0, four); in ITransformSSE2()
185 const __m128i a = _mm_add_epi16(dc, T2); in ITransformSSE2()
186 const __m128i b = _mm_sub_epi16(dc, T2); in ITransformSSE2()
188 const __m128i c1 = _mm_mulhi_epi16(T1, k2); in ITransformSSE2()
189 const __m128i c2 = _mm_mulhi_epi16(T3, k1); in ITransformSSE2()
190 const __m128i c3 = _mm_sub_epi16(T1, T3); in ITransformSSE2()
191 const __m128i c4 = _mm_sub_epi16(c1, c2); in ITransformSSE2()
192 const __m128i c = _mm_add_epi16(c3, c4); in ITransformSSE2()
194 const __m128i d1 = _mm_mulhi_epi16(T1, k1); in ITransformSSE2()
195 const __m128i d2 = _mm_mulhi_epi16(T3, k2); in ITransformSSE2()
196 const __m128i d3 = _mm_add_epi16(T1, T3); in ITransformSSE2()
197 const __m128i d4 = _mm_add_epi16(d1, d2); in ITransformSSE2()
198 const __m128i d = _mm_add_epi16(d3, d4); in ITransformSSE2()
201 const __m128i tmp0 = _mm_add_epi16(a, d); in ITransformSSE2()
202 const __m128i tmp1 = _mm_add_epi16(b, c); in ITransformSSE2()
203 const __m128i tmp2 = _mm_sub_epi16(b, c); in ITransformSSE2()
204 const __m128i tmp3 = _mm_sub_epi16(a, d); in ITransformSSE2()
205 const __m128i shifted0 = _mm_srai_epi16(tmp0, 3); in ITransformSSE2()
206 const __m128i shifted1 = _mm_srai_epi16(tmp1, 3); in ITransformSSE2()
207 const __m128i shifted2 = _mm_srai_epi16(tmp2, 3); in ITransformSSE2()
208 const __m128i shifted3 = _mm_srai_epi16(tmp3, 3); in ITransformSSE2()
215 const __m128i transpose0_0 = _mm_unpacklo_epi16(shifted0, shifted1); in ITransformSSE2()
216 const __m128i transpose0_1 = _mm_unpacklo_epi16(shifted2, shifted3); in ITransformSSE2()
217 const __m128i transpose0_2 = _mm_unpackhi_epi16(shifted0, shifted1); in ITransformSSE2()
218 const __m128i transpose0_3 = _mm_unpackhi_epi16(shifted2, shifted3); in ITransformSSE2()
223 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); in ITransformSSE2()
224 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); in ITransformSSE2()
225 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); in ITransformSSE2()
226 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); in ITransformSSE2()
243 const __m128i zero = _mm_set1_epi16(0); in ITransformSSE2()
245 __m128i ref0, ref1, ref2, ref3; in ITransformSSE2()
248 ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]); in ITransformSSE2()
249 ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]); in ITransformSSE2()
250 ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]); in ITransformSSE2()
251 ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]); in ITransformSSE2()
277 _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); in ITransformSSE2()
278 _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); in ITransformSSE2()
279 _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); in ITransformSSE2()
280 _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); in ITransformSSE2()
293 const __m128i zero = _mm_setzero_si128(); in FTransformSSE2()
294 const __m128i seven = _mm_set1_epi16(7); in FTransformSSE2()
295 const __m128i k7500 = _mm_set1_epi32(7500); in FTransformSSE2()
296 const __m128i k14500 = _mm_set1_epi32(14500); in FTransformSSE2()
297 const __m128i k51000 = _mm_set1_epi32(51000); in FTransformSSE2()
298 const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16)); in FTransformSSE2()
299 const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217, in FTransformSSE2()
301 const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352, in FTransformSSE2()
304 __m128i v01, v32; in FTransformSSE2()
309 const __m128i src0 = _mm_loadl_epi64((__m128i*)&src[0 * BPS]); in FTransformSSE2()
310 const __m128i src1 = _mm_loadl_epi64((__m128i*)&src[1 * BPS]); in FTransformSSE2()
311 const __m128i src2 = _mm_loadl_epi64((__m128i*)&src[2 * BPS]); in FTransformSSE2()
312 const __m128i src3 = _mm_loadl_epi64((__m128i*)&src[3 * BPS]); in FTransformSSE2()
313 const __m128i src_0 = _mm_unpacklo_epi8(src0, zero); in FTransformSSE2()
314 const __m128i src_1 = _mm_unpacklo_epi8(src1, zero); in FTransformSSE2()
315 const __m128i src_2 = _mm_unpacklo_epi8(src2, zero); in FTransformSSE2()
316 const __m128i src_3 = _mm_unpacklo_epi8(src3, zero); in FTransformSSE2()
318 const __m128i ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]); in FTransformSSE2()
319 const __m128i ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]); in FTransformSSE2()
320 const __m128i ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]); in FTransformSSE2()
321 const __m128i ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]); in FTransformSSE2()
322 const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero); in FTransformSSE2()
323 const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero); in FTransformSSE2()
324 const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero); in FTransformSSE2()
325 const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero); in FTransformSSE2()
327 const __m128i diff0 = _mm_sub_epi16(src_0, ref_0); in FTransformSSE2()
328 const __m128i diff1 = _mm_sub_epi16(src_1, ref_1); in FTransformSSE2()
329 const __m128i diff2 = _mm_sub_epi16(src_2, ref_2); in FTransformSSE2()
330 const __m128i diff3 = _mm_sub_epi16(src_3, ref_3); in FTransformSSE2()
337 const __m128i transpose0_0 = _mm_unpacklo_epi16(diff0, diff1); in FTransformSSE2()
338 const __m128i transpose0_1 = _mm_unpacklo_epi16(diff2, diff3); in FTransformSSE2()
341 const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); in FTransformSSE2()
356 const __m128i a01 = _mm_add_epi16(v01, v32); in FTransformSSE2()
357 const __m128i a32 = _mm_sub_epi16(v01, v32); in FTransformSSE2()
358 const __m128i b01 = _mm_slli_epi16(a01, 3); in FTransformSSE2()
359 const __m128i b32 = _mm_slli_epi16(a32, 3); in FTransformSSE2()
360 const __m128i b11 = _mm_unpackhi_epi64(b01, b01); in FTransformSSE2()
361 const __m128i b22 = _mm_unpackhi_epi64(b32, b32); in FTransformSSE2()
365 const __m128i e0 = _mm_add_epi16(b01, b11); in FTransformSSE2()
366 const __m128i e2 = _mm_sub_epi16(b01, b11); in FTransformSSE2()
367 const __m128i e02 = _mm_unpacklo_epi64(e0, e2); in FTransformSSE2()
371 const __m128i b23 = _mm_unpacklo_epi16(b22, b32); in FTransformSSE2()
372 const __m128i c1 = _mm_madd_epi16(b23, k5352_2217); in FTransformSSE2()
373 const __m128i c3 = _mm_madd_epi16(b23, k2217_5352); in FTransformSSE2()
374 const __m128i d1 = _mm_add_epi32(c1, k14500); in FTransformSSE2()
375 const __m128i d3 = _mm_add_epi32(c3, k7500); in FTransformSSE2()
376 const __m128i e1 = _mm_srai_epi32(d1, 12); in FTransformSSE2()
377 const __m128i e3 = _mm_srai_epi32(d3, 12); in FTransformSSE2()
378 const __m128i e13 = _mm_packs_epi32(e1, e3); in FTransformSSE2()
383 const __m128i transpose0_0 = _mm_unpacklo_epi16(e02, e13); in FTransformSSE2()
384 const __m128i transpose0_1 = _mm_unpackhi_epi16(e02, e13); in FTransformSSE2()
387 const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); in FTransformSSE2()
402 const __m128i a01 = _mm_add_epi16(v01, v32); in FTransformSSE2()
403 const __m128i a32 = _mm_sub_epi16(v01, v32); in FTransformSSE2()
404 const __m128i a11 = _mm_unpackhi_epi64(a01, a01); in FTransformSSE2()
405 const __m128i a22 = _mm_unpackhi_epi64(a32, a32); in FTransformSSE2()
409 const __m128i b0 = _mm_add_epi16(a01, a11); in FTransformSSE2()
410 const __m128i b2 = _mm_sub_epi16(a01, a11); in FTransformSSE2()
411 const __m128i c0 = _mm_add_epi16(b0, seven); in FTransformSSE2()
412 const __m128i c2 = _mm_add_epi16(b2, seven); in FTransformSSE2()
413 const __m128i d0 = _mm_srai_epi16(c0, 4); in FTransformSSE2()
414 const __m128i d2 = _mm_srai_epi16(c2, 4); in FTransformSSE2()
418 const __m128i b23 = _mm_unpacklo_epi16(a22, a32); in FTransformSSE2()
419 const __m128i c1 = _mm_madd_epi16(b23, k5352_2217); in FTransformSSE2()
420 const __m128i c3 = _mm_madd_epi16(b23, k2217_5352); in FTransformSSE2()
421 const __m128i d1 = _mm_add_epi32(c1, k12000_plus_one); in FTransformSSE2()
422 const __m128i d3 = _mm_add_epi32(c3, k51000); in FTransformSSE2()
423 const __m128i e1 = _mm_srai_epi32(d1, 16); in FTransformSSE2()
424 const __m128i e3 = _mm_srai_epi32(d3, 16); in FTransformSSE2()
425 const __m128i f1 = _mm_packs_epi32(e1, e1); in FTransformSSE2()
426 const __m128i f3 = _mm_packs_epi32(e3, e3); in FTransformSSE2()
430 const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero)); in FTransformSSE2()
432 _mm_storel_epi64((__m128i*)&out[ 0], d0); in FTransformSSE2()
433 _mm_storel_epi64((__m128i*)&out[ 4], g1); in FTransformSSE2()
434 _mm_storel_epi64((__m128i*)&out[ 8], d2); in FTransformSSE2()
435 _mm_storel_epi64((__m128i*)&out[12], f3); in FTransformSSE2()
443 const __m128i zero = _mm_set1_epi16(0); in SSE4x4SSE2()
446 const __m128i a0 = _mm_loadl_epi64((__m128i*)&a[BPS * 0]); in SSE4x4SSE2()
447 const __m128i a1 = _mm_loadl_epi64((__m128i*)&a[BPS * 1]); in SSE4x4SSE2()
448 const __m128i a2 = _mm_loadl_epi64((__m128i*)&a[BPS * 2]); in SSE4x4SSE2()
449 const __m128i a3 = _mm_loadl_epi64((__m128i*)&a[BPS * 3]); in SSE4x4SSE2()
450 const __m128i b0 = _mm_loadl_epi64((__m128i*)&b[BPS * 0]); in SSE4x4SSE2()
451 const __m128i b1 = _mm_loadl_epi64((__m128i*)&b[BPS * 1]); in SSE4x4SSE2()
452 const __m128i b2 = _mm_loadl_epi64((__m128i*)&b[BPS * 2]); in SSE4x4SSE2()
453 const __m128i b3 = _mm_loadl_epi64((__m128i*)&b[BPS * 3]); in SSE4x4SSE2()
456 const __m128i a01 = _mm_unpacklo_epi32(a0, a1); in SSE4x4SSE2()
457 const __m128i a23 = _mm_unpacklo_epi32(a2, a3); in SSE4x4SSE2()
458 const __m128i b01 = _mm_unpacklo_epi32(b0, b1); in SSE4x4SSE2()
459 const __m128i b23 = _mm_unpacklo_epi32(b2, b3); in SSE4x4SSE2()
460 const __m128i a01s = _mm_unpacklo_epi8(a01, zero); in SSE4x4SSE2()
461 const __m128i a23s = _mm_unpacklo_epi8(a23, zero); in SSE4x4SSE2()
462 const __m128i b01s = _mm_unpacklo_epi8(b01, zero); in SSE4x4SSE2()
463 const __m128i b23s = _mm_unpacklo_epi8(b23, zero); in SSE4x4SSE2()
470 const __m128i d0 = _mm_subs_epu8(a01s, b01s); in SSE4x4SSE2()
471 const __m128i d1 = _mm_subs_epu8(b01s, a01s); in SSE4x4SSE2()
472 const __m128i d2 = _mm_subs_epu8(a23s, b23s); in SSE4x4SSE2()
473 const __m128i d3 = _mm_subs_epu8(b23s, a23s); in SSE4x4SSE2()
476 const __m128i madd0 = _mm_madd_epi16(d0, d0); in SSE4x4SSE2()
477 const __m128i madd1 = _mm_madd_epi16(d1, d1); in SSE4x4SSE2()
478 const __m128i madd2 = _mm_madd_epi16(d2, d2); in SSE4x4SSE2()
479 const __m128i madd3 = _mm_madd_epi16(d3, d3); in SSE4x4SSE2()
480 const __m128i sum0 = _mm_add_epi32(madd0, madd1); in SSE4x4SSE2()
481 const __m128i sum1 = _mm_add_epi32(madd2, madd3); in SSE4x4SSE2()
482 const __m128i sum2 = _mm_add_epi32(sum0, sum1); in SSE4x4SSE2()
484 _mm_storeu_si128((__m128i*)tmp, sum2); in SSE4x4SSE2()
500 __m128i tmp_0, tmp_1, tmp_2, tmp_3; in TTransformSSE2()
501 const __m128i zero = _mm_setzero_si128(); in TTransformSSE2()
502 const __m128i one = _mm_set1_epi16(1); in TTransformSSE2()
503 const __m128i three = _mm_set1_epi16(3); in TTransformSSE2()
507 const __m128i inA_0 = _mm_loadl_epi64((__m128i*)&inA[BPS * 0]); in TTransformSSE2()
508 const __m128i inA_1 = _mm_loadl_epi64((__m128i*)&inA[BPS * 1]); in TTransformSSE2()
509 const __m128i inA_2 = _mm_loadl_epi64((__m128i*)&inA[BPS * 2]); in TTransformSSE2()
510 const __m128i inA_3 = _mm_loadl_epi64((__m128i*)&inA[BPS * 3]); in TTransformSSE2()
511 const __m128i inB_0 = _mm_loadl_epi64((__m128i*)&inB[BPS * 0]); in TTransformSSE2()
512 const __m128i inB_1 = _mm_loadl_epi64((__m128i*)&inB[BPS * 1]); in TTransformSSE2()
513 const __m128i inB_2 = _mm_loadl_epi64((__m128i*)&inB[BPS * 2]); in TTransformSSE2()
514 const __m128i inB_3 = _mm_loadl_epi64((__m128i*)&inB[BPS * 3]); in TTransformSSE2()
517 const __m128i inAB_0 = _mm_unpacklo_epi8(inA_0, inB_0); in TTransformSSE2()
518 const __m128i inAB_1 = _mm_unpacklo_epi8(inA_1, inB_1); in TTransformSSE2()
519 const __m128i inAB_2 = _mm_unpacklo_epi8(inA_2, inB_2); in TTransformSSE2()
520 const __m128i inAB_3 = _mm_unpacklo_epi8(inA_3, inB_3); in TTransformSSE2()
527 const __m128i transpose0_0 = _mm_unpacklo_epi8(inAB_0, inAB_2); in TTransformSSE2()
528 const __m128i transpose0_1 = _mm_unpacklo_epi8(inAB_1, inAB_3); in TTransformSSE2()
531 const __m128i transpose1_0 = _mm_unpacklo_epi8(transpose0_0, transpose0_1); in TTransformSSE2()
532 const __m128i transpose1_1 = _mm_unpackhi_epi8(transpose0_0, transpose0_1); in TTransformSSE2()
550 const __m128i a0 = _mm_slli_epi16(_mm_add_epi16(tmp_0, tmp_2), 2); in TTransformSSE2()
551 const __m128i a1 = _mm_slli_epi16(_mm_add_epi16(tmp_1, tmp_3), 2); in TTransformSSE2()
552 const __m128i a2 = _mm_slli_epi16(_mm_sub_epi16(tmp_1, tmp_3), 2); in TTransformSSE2()
553 const __m128i a3 = _mm_slli_epi16(_mm_sub_epi16(tmp_0, tmp_2), 2); in TTransformSSE2()
555 const __m128i b0_extra = _mm_andnot_si128(_mm_cmpeq_epi16 (a0, zero), one); in TTransformSSE2()
556 const __m128i b0_base = _mm_add_epi16(a0, a1); in TTransformSSE2()
557 const __m128i b1 = _mm_add_epi16(a3, a2); in TTransformSSE2()
558 const __m128i b2 = _mm_sub_epi16(a3, a2); in TTransformSSE2()
559 const __m128i b3 = _mm_sub_epi16(a0, a1); in TTransformSSE2()
560 const __m128i b0 = _mm_add_epi16(b0_base, b0_extra); in TTransformSSE2()
567 const __m128i transpose0_0 = _mm_unpacklo_epi16(b0, b1); in TTransformSSE2()
568 const __m128i transpose0_1 = _mm_unpacklo_epi16(b2, b3); in TTransformSSE2()
569 const __m128i transpose0_2 = _mm_unpackhi_epi16(b0, b1); in TTransformSSE2()
570 const __m128i transpose0_3 = _mm_unpackhi_epi16(b2, b3); in TTransformSSE2()
575 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); in TTransformSSE2()
576 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); in TTransformSSE2()
577 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); in TTransformSSE2()
578 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); in TTransformSSE2()
598 const __m128i w_0 = _mm_loadu_si128((__m128i*)&w[0]); in TTransformSSE2()
599 const __m128i w_8 = _mm_loadu_si128((__m128i*)&w[8]); in TTransformSSE2()
602 const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); in TTransformSSE2()
603 const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); in TTransformSSE2()
604 const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); in TTransformSSE2()
605 const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); in TTransformSSE2()
606 const __m128i b0 = _mm_add_epi16(a0, a1); in TTransformSSE2()
607 const __m128i b1 = _mm_add_epi16(a3, a2); in TTransformSSE2()
608 const __m128i b2 = _mm_sub_epi16(a3, a2); in TTransformSSE2()
609 const __m128i b3 = _mm_sub_epi16(a0, a1); in TTransformSSE2()
612 __m128i A_b0 = _mm_unpacklo_epi64(b0, b1); in TTransformSSE2()
613 __m128i A_b2 = _mm_unpacklo_epi64(b2, b3); in TTransformSSE2()
614 __m128i B_b0 = _mm_unpackhi_epi64(b0, b1); in TTransformSSE2()
615 __m128i B_b2 = _mm_unpackhi_epi64(b2, b3); in TTransformSSE2()
619 const __m128i sign_A_b0 = _mm_srai_epi16(A_b0, 15); in TTransformSSE2()
620 const __m128i sign_A_b2 = _mm_srai_epi16(A_b2, 15); in TTransformSSE2()
621 const __m128i sign_B_b0 = _mm_srai_epi16(B_b0, 15); in TTransformSSE2()
622 const __m128i sign_B_b2 = _mm_srai_epi16(B_b2, 15); in TTransformSSE2()
658 _mm_storeu_si128((__m128i*)&sum[0], A_b0); in TTransformSSE2()
689 const __m128i max_coeff_2047 = _mm_set1_epi16(2047); in QuantizeBlockSSE2()
690 const __m128i zero = _mm_set1_epi16(0); in QuantizeBlockSSE2()
691 __m128i sign0, sign8; in QuantizeBlockSSE2()
692 __m128i coeff0, coeff8; in QuantizeBlockSSE2()
693 __m128i out0, out8; in QuantizeBlockSSE2()
694 __m128i packed_out; in QuantizeBlockSSE2()
699 __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); in QuantizeBlockSSE2()
700 __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); in QuantizeBlockSSE2()
701 const __m128i sharpen0 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[0]); in QuantizeBlockSSE2()
702 const __m128i sharpen8 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[8]); in QuantizeBlockSSE2()
703 const __m128i iq0 = _mm_loadu_si128((__m128i*)&mtx->iq_[0]); in QuantizeBlockSSE2()
704 const __m128i iq8 = _mm_loadu_si128((__m128i*)&mtx->iq_[8]); in QuantizeBlockSSE2()
705 const __m128i bias0 = _mm_loadu_si128((__m128i*)&mtx->bias_[0]); in QuantizeBlockSSE2()
706 const __m128i bias8 = _mm_loadu_si128((__m128i*)&mtx->bias_[8]); in QuantizeBlockSSE2()
707 const __m128i q0 = _mm_loadu_si128((__m128i*)&mtx->q_[0]); in QuantizeBlockSSE2()
708 const __m128i q8 = _mm_loadu_si128((__m128i*)&mtx->q_[8]); in QuantizeBlockSSE2()
709 const __m128i zthresh0 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[0]); in QuantizeBlockSSE2()
710 const __m128i zthresh8 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[8]); in QuantizeBlockSSE2()
734 __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0); in QuantizeBlockSSE2()
735 __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0); in QuantizeBlockSSE2()
736 __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8); in QuantizeBlockSSE2()
737 __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8); in QuantizeBlockSSE2()
738 __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H); in QuantizeBlockSSE2()
739 __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H); in QuantizeBlockSSE2()
740 __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H); in QuantizeBlockSSE2()
741 __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H); in QuantizeBlockSSE2()
743 __m128i bias_00 = _mm_unpacklo_epi16(bias0, zero); in QuantizeBlockSSE2()
744 __m128i bias_04 = _mm_unpackhi_epi16(bias0, zero); in QuantizeBlockSSE2()
745 __m128i bias_08 = _mm_unpacklo_epi16(bias8, zero); in QuantizeBlockSSE2()
746 __m128i bias_12 = _mm_unpackhi_epi16(bias8, zero); in QuantizeBlockSSE2()
774 __m128i cmp0 = _mm_cmpgt_epi16(coeff0, zthresh0); in QuantizeBlockSSE2()
775 __m128i cmp8 = _mm_cmpgt_epi16(coeff8, zthresh8); in QuantizeBlockSSE2()
778 _mm_storeu_si128((__m128i*)&in[0], in0); in QuantizeBlockSSE2()
779 _mm_storeu_si128((__m128i*)&in[8], in8); in QuantizeBlockSSE2()
790 __m128i outZ0, outZ8; in QuantizeBlockSSE2()
797 _mm_storeu_si128((__m128i*)&out[0], outZ0); in QuantizeBlockSSE2()
798 _mm_storeu_si128((__m128i*)&out[8], outZ8); in QuantizeBlockSSE2()
811 _mm_storeu_si128((__m128i*)tmp, packed_out); in QuantizeBlockSSE2()