Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qdrawhelper_avx2.cpp
Go to the documentation of this file.
1// Copyright (C) 2018 The Qt Company Ltd.
2// Copyright (C) 2018 Intel Corporation.
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
4// Qt-Security score:significant reason:default
5
6#include "qdrawhelper_p.h"
7#include "qdrawhelper_x86_p.h"
8#include "qdrawingprimitive_sse2_p.h"
9#include "qpixellayout_p.h"
10#include "qrgba64_p.h"
11
12#if defined(QT_COMPILER_SUPPORTS_AVX2)
13
14QT_BEGIN_NAMESPACE
15
16enum {
17 FixedScale = 1 << 16,
18 HalfPoint = 1 << 15
19};
20
21// Vectorized blend functions:
22
23// See BYTE_MUL_SSE2 for details.
24inline static void Q_DECL_VECTORCALL
25BYTE_MUL_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
26{
27 __m256i pixelVectorAG = _mm256_srli_epi16(pixelVector, 8);
28 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
29
30 pixelVectorAG = _mm256_mullo_epi16(pixelVectorAG, alphaChannel);
31 pixelVectorRB = _mm256_mullo_epi16(pixelVectorRB, alphaChannel);
32
33 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, _mm256_srli_epi16(pixelVectorRB, 8));
34 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, _mm256_srli_epi16(pixelVectorAG, 8));
35 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, half);
36 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, half);
37
38 pixelVectorRB = _mm256_srli_epi16(pixelVectorRB, 8);
39 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
40}
41
42#if QT_CONFIG(raster_64bit)
43inline static void Q_DECL_VECTORCALL
44BYTE_MUL_RGB64_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
45{
46 __m256i pixelVectorAG = _mm256_srli_epi32(pixelVector, 16);
47 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
48
49 pixelVectorAG = _mm256_mullo_epi32(pixelVectorAG, alphaChannel);
50 pixelVectorRB = _mm256_mullo_epi32(pixelVectorRB, alphaChannel);
51
52 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, _mm256_srli_epi32(pixelVectorRB, 16));
53 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, _mm256_srli_epi32(pixelVectorAG, 16));
54 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, half);
55 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, half);
56
57 pixelVectorRB = _mm256_srli_epi32(pixelVectorRB, 16);
58 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
59}
60#endif
61
62// See INTERPOLATE_PIXEL_255_SSE2 for details.
63inline static void Q_DECL_VECTORCALL
64INTERPOLATE_PIXEL_255_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
65{
66 const __m256i srcVectorAG = _mm256_srli_epi16(srcVector, 8);
67 const __m256i dstVectorAG = _mm256_srli_epi16(dstVector, 8);
68 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
69 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
70 const __m256i srcVectorAGalpha = _mm256_mullo_epi16(srcVectorAG, alphaChannel);
71 const __m256i srcVectorRBalpha = _mm256_mullo_epi16(srcVectorRB, alphaChannel);
72 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi16(dstVectorAG, oneMinusAlphaChannel);
73 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi16(dstVectorRB, oneMinusAlphaChannel);
74 __m256i finalAG = _mm256_add_epi16(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
75 __m256i finalRB = _mm256_add_epi16(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
76 finalAG = _mm256_add_epi16(finalAG, _mm256_srli_epi16(finalAG, 8));
77 finalRB = _mm256_add_epi16(finalRB, _mm256_srli_epi16(finalRB, 8));
78 finalAG = _mm256_add_epi16(finalAG, half);
79 finalRB = _mm256_add_epi16(finalRB, half);
80 finalRB = _mm256_srli_epi16(finalRB, 8);
81
82 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
83}
84
85#if QT_CONFIG(raster_64bit)
86inline static void Q_DECL_VECTORCALL
87INTERPOLATE_PIXEL_RGB64_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
88{
89 const __m256i srcVectorAG = _mm256_srli_epi32(srcVector, 16);
90 const __m256i dstVectorAG = _mm256_srli_epi32(dstVector, 16);
91 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
92 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
93 const __m256i srcVectorAGalpha = _mm256_mullo_epi32(srcVectorAG, alphaChannel);
94 const __m256i srcVectorRBalpha = _mm256_mullo_epi32(srcVectorRB, alphaChannel);
95 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi32(dstVectorAG, oneMinusAlphaChannel);
96 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi32(dstVectorRB, oneMinusAlphaChannel);
97 __m256i finalAG = _mm256_add_epi32(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
98 __m256i finalRB = _mm256_add_epi32(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
99 finalAG = _mm256_add_epi32(finalAG, _mm256_srli_epi32(finalAG, 16));
100 finalRB = _mm256_add_epi32(finalRB, _mm256_srli_epi32(finalRB, 16));
101 finalAG = _mm256_add_epi32(finalAG, half);
102 finalRB = _mm256_add_epi32(finalRB, half);
103 finalRB = _mm256_srli_epi32(finalRB, 16);
104 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
105}
106#endif
107
108// See BLEND_SOURCE_OVER_ARGB32_SSE2 for details.
109inline static void Q_DECL_VECTORCALL BLEND_SOURCE_OVER_ARGB32_AVX2(quint32 *dst, const quint32 *src, const int length)
110{
111 const __m256i half = _mm256_set1_epi16(0x80);
112 const __m256i one = _mm256_set1_epi16(0xff);
113 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
114 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
115 const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
116 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
117 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
118
119 const int minusOffsetToAlignDstOn32Bytes = (reinterpret_cast<quintptr>(dst) >> 2) & 0x7;
120
121 int x = 0;
122 // Prologue to handle all pixels until dst is 32-byte aligned in one step.
123 if (minusOffsetToAlignDstOn32Bytes != 0 && x < (length - 7)) {
124 const __m256i prologueMask = _mm256_sub_epi32(_mm256_set1_epi32(minusOffsetToAlignDstOn32Bytes - 1), offsetMask);
125 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
126 const __m256i prologueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, prologueMask);
127 if (!_mm256_testz_si256(srcVector, prologueAlphaMask)) {
128 if (_mm256_testc_si256(srcVector, prologueAlphaMask)) {
129 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, srcVector);
130 } else {
131 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
132 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
133 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
134 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
135 dstVector = _mm256_add_epi8(dstVector, srcVector);
136 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, dstVector);
137 }
138 }
139 x += (8 - minusOffsetToAlignDstOn32Bytes);
140 }
141
142 for (; x < (length - 7); x += 8) {
143 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
144 if (!_mm256_testz_si256(srcVector, alphaMask)) {
145 if (_mm256_testc_si256(srcVector, alphaMask)) {
146 _mm256_store_si256((__m256i *)&dst[x], srcVector);
147 } else {
148 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
149 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
150 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
151 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
152 dstVector = _mm256_add_epi8(dstVector, srcVector);
153 _mm256_store_si256((__m256i *)&dst[x], dstVector);
154 }
155 }
156 }
157
158 // Epilogue to handle all remaining pixels in one step.
159 if (x < length) {
160 const __m256i epilogueMask = _mm256_add_epi32(offsetMask, _mm256_set1_epi32(x - length));
161 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x], epilogueMask);
162 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
163 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
164 if (_mm256_testc_si256(srcVector, epilogueAlphaMask)) {
165 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, srcVector);
166 } else {
167 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
168 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
169 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x], epilogueMask);
170 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
171 dstVector = _mm256_add_epi8(dstVector, srcVector);
172 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, dstVector);
173 }
174 }
175 }
176}
177
178
179// See BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_SSE2 for details.
180inline static void Q_DECL_VECTORCALL
181BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(quint32 *dst, const quint32 *src, const int length, const int const_alpha)
182{
183 int x = 0;
184
185 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
186 blend_pixel(dst[x], src[x], const_alpha);
187
188 const __m256i half = _mm256_set1_epi16(0x80);
189 const __m256i one = _mm256_set1_epi16(0xff);
190 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
191 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
192 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
193 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
194 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
195 for (; x < (length - 7); x += 8) {
196 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
197 if (!_mm256_testz_si256(srcVector, alphaMask)) {
198 BYTE_MUL_AVX2(srcVector, constAlphaVector, colorMask, half);
199
200 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
201 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
202 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
203 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
204 dstVector = _mm256_add_epi8(dstVector, srcVector);
205 _mm256_store_si256((__m256i *)&dst[x], dstVector);
206 }
207 }
208 SIMD_EPILOGUE(x, length, 7)
209 blend_pixel(dst[x], src[x], const_alpha);
210}
211
212void qt_blend_argb32_on_argb32_avx2(uchar *destPixels, int dbpl,
213 const uchar *srcPixels, int sbpl,
214 int w, int h,
215 int const_alpha)
216{
217 if (const_alpha == 256) {
218 for (int y = 0; y < h; ++y) {
219 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
220 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
221 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, w);
222 destPixels += dbpl;
223 srcPixels += sbpl;
224 }
225 } else if (const_alpha != 0) {
226 const_alpha = (const_alpha * 255) >> 8;
227 for (int y = 0; y < h; ++y) {
228 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
229 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
230 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, w, const_alpha);
231 destPixels += dbpl;
232 srcPixels += sbpl;
233 }
234 }
235}
236
237void qt_blend_rgb32_on_rgb32_avx2(uchar *destPixels, int dbpl,
238 const uchar *srcPixels, int sbpl,
239 int w, int h,
240 int const_alpha)
241{
242 if (const_alpha == 256) {
243 for (int y = 0; y < h; ++y) {
244 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
245 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
246 ::memcpy(dst, src, w * sizeof(uint));
247 srcPixels += sbpl;
248 destPixels += dbpl;
249 }
250 return;
251 }
252 if (const_alpha == 0)
253 return;
254
255 const __m256i half = _mm256_set1_epi16(0x80);
256 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
257
258 const_alpha = (const_alpha * 255) >> 8;
259 int one_minus_const_alpha = 255 - const_alpha;
260 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
261 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(one_minus_const_alpha);
262 for (int y = 0; y < h; ++y) {
263 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
264 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
265 int x = 0;
266
267 // First, align dest to 32 bytes:
268 ALIGNMENT_PROLOGUE_32BYTES(dst, x, w)
269 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
270
271 // 2) interpolate pixels with AVX2
272 for (; x < (w - 7); x += 8) {
273 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
274 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
275 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
276 _mm256_store_si256((__m256i *)&dst[x], dstVector);
277 }
278
279 // 3) Epilogue
280 SIMD_EPILOGUE(x, w, 7)
281 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
282
283 srcPixels += sbpl;
284 destPixels += dbpl;
285 }
286}
287
288Q_NEVER_INLINE static
289void Q_DECL_VECTORCALL qt_memfillXX_avx2(uchar *dest, __m256i value256, qsizetype bytes)
290{
291 __m128i value128 = _mm256_castsi256_si128(value256);
292
293 // main body
294 __m256i *dst256 = reinterpret_cast<__m256i *>(dest);
295 uchar *end = dest + bytes;
296 while (reinterpret_cast<uchar *>(dst256 + 4) <= end) {
297 _mm256_storeu_si256(dst256 + 0, value256);
298 _mm256_storeu_si256(dst256 + 1, value256);
299 _mm256_storeu_si256(dst256 + 2, value256);
300 _mm256_storeu_si256(dst256 + 3, value256);
301 dst256 += 4;
302 }
303
304 // first epilogue: fewer than 128 bytes / 32 entries
305 bytes = end - reinterpret_cast<uchar *>(dst256);
306 switch (bytes / sizeof(value256)) {
307 case 3: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
308 case 2: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
309 case 1: _mm256_storeu_si256(dst256++, value256);
310 }
311
312 // second epilogue: fewer than 32 bytes
313 __m128i *dst128 = reinterpret_cast<__m128i *>(dst256);
314 if (bytes & sizeof(value128))
315 _mm_storeu_si128(dst128++, value128);
316
317 // third epilogue: fewer than 16 bytes
318 if (bytes & 8)
319 _mm_storel_epi64(reinterpret_cast<__m128i *>(end - 8), value128);
320}
321
322void qt_memfill64_avx2(quint64 *dest, quint64 value, qsizetype count)
323{
324#if defined(Q_CC_GNU) && !defined(Q_CC_CLANG)
325 // work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80820
326 __m128i value64 = _mm_set_epi64x(0, value); // _mm_cvtsi64_si128(value);
327# ifdef Q_PROCESSOR_X86_64
328 asm ("" : "+x" (value64));
329# endif
330 __m256i value256 = _mm256_broadcastq_epi64(value64);
331#else
332 __m256i value256 = _mm256_set1_epi64x(value);
333#endif
334
335 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), value256, count * sizeof(quint64));
336}
337
338void qt_memfill32_avx2(quint32 *dest, quint32 value, qsizetype count)
339{
340 if (count % 2) {
341 // odd number of pixels, round to even
342 *dest++ = value;
343 --count;
344 }
345 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), _mm256_set1_epi32(value), count * sizeof(quint32));
346}
347
348void QT_FASTCALL comp_func_SourceOver_avx2(uint *destPixels, const uint *srcPixels, int length, uint const_alpha)
349{
350 Q_ASSERT(const_alpha < 256);
351
352 const quint32 *src = (const quint32 *) srcPixels;
353 quint32 *dst = (quint32 *) destPixels;
354
355 if (const_alpha == 255)
356 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, length);
357 else
358 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, length, const_alpha);
359}
360
361#if QT_CONFIG(raster_64bit)
362void QT_FASTCALL comp_func_SourceOver_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
363{
364 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
365 const __m256i half = _mm256_set1_epi32(0x8000);
366 const __m256i one = _mm256_set1_epi32(0xffff);
367 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
368 __m256i alphaMask = _mm256_set1_epi32(0xff000000);
369 alphaMask = _mm256_unpacklo_epi8(alphaMask, alphaMask);
370 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6,
371 char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6);
372
373 if (const_alpha == 255) {
374 int x = 0;
375 for (; x < length && (quintptr(dst + x) & 31); ++x)
376 blend_pixel(dst[x], src[x]);
377 for (; x < length - 3; x += 4) {
378 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
379 if (!_mm256_testz_si256(srcVector, alphaMask)) {
380 // Not all transparent
381 if (_mm256_testc_si256(srcVector, alphaMask)) {
382 // All opaque
383 _mm256_store_si256((__m256i *)&dst[x], srcVector);
384 } else {
385 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
386 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
387 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
388 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
389 dstVector = _mm256_add_epi16(dstVector, srcVector);
390 _mm256_store_si256((__m256i *)&dst[x], dstVector);
391 }
392 }
393 }
394 SIMD_EPILOGUE(x, length, 3)
395 blend_pixel(dst[x], src[x]);
396 } else {
397 const __m256i constAlphaVector = _mm256_set1_epi32(const_alpha | (const_alpha << 8));
398 int x = 0;
399 for (; x < length && (quintptr(dst + x) & 31); ++x)
400 blend_pixel(dst[x], src[x], const_alpha);
401 for (; x < length - 3; x += 4) {
402 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
403 if (!_mm256_testz_si256(srcVector, alphaMask)) {
404 // Not all transparent
405 BYTE_MUL_RGB64_AVX2(srcVector, constAlphaVector, colorMask, half);
406
407 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
408 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
409 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
410 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
411 dstVector = _mm256_add_epi16(dstVector, srcVector);
412 _mm256_store_si256((__m256i *)&dst[x], dstVector);
413 }
414 }
415 SIMD_EPILOGUE(x, length, 3)
416 blend_pixel(dst[x], src[x], const_alpha);
417 }
418}
419#endif
420
421#if QT_CONFIG(raster_fp)
422void QT_FASTCALL comp_func_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
423{
424 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
425
426 const float a = const_alpha / 255.0f;
427 const __m128 one = _mm_set1_ps(1.0f);
428 const __m128 constAlphaVector = _mm_set1_ps(a);
429 const __m256 one256 = _mm256_set1_ps(1.0f);
430 const __m256 constAlphaVector256 = _mm256_set1_ps(a);
431 int x = 0;
432 for (; x < length - 1; x += 2) {
433 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
434 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
435 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
436 __m256 alphaChannel = _mm256_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
437 alphaChannel = _mm256_sub_ps(one256, alphaChannel);
438 dstVector = _mm256_mul_ps(dstVector, alphaChannel);
439 dstVector = _mm256_add_ps(dstVector, srcVector);
440 _mm256_storeu_ps((float *)(dst + x), dstVector);
441 }
442 if (x < length) {
443 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
444 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
445 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
446 __m128 alphaChannel = _mm_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
447 alphaChannel = _mm_sub_ps(one, alphaChannel);
448 dstVector = _mm_mul_ps(dstVector, alphaChannel);
449 dstVector = _mm_add_ps(dstVector, srcVector);
450 _mm_storeu_ps((float *)(dst + x), dstVector);
451 }
452}
453#endif
454
455void QT_FASTCALL comp_func_Source_avx2(uint *dst, const uint *src, int length, uint const_alpha)
456{
457 if (const_alpha == 255) {
458 ::memcpy(dst, src, length * sizeof(uint));
459 } else {
460 const int ialpha = 255 - const_alpha;
461
462 int x = 0;
463
464 // 1) prologue, align on 32 bytes
465 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
466 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
467
468 // 2) interpolate pixels with AVX2
469 const __m256i half = _mm256_set1_epi16(0x80);
470 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
471 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
472 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(ialpha);
473 for (; x < length - 7; x += 8) {
474 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
475 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
476 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
477 _mm256_store_si256((__m256i *)&dst[x], dstVector);
478 }
479
480 // 3) Epilogue
481 SIMD_EPILOGUE(x, length, 7)
482 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
483 }
484}
485
486#if QT_CONFIG(raster_64bit)
487void QT_FASTCALL comp_func_Source_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
488{
489 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
490 if (const_alpha == 255) {
491 ::memcpy(dst, src, length * sizeof(QRgba64));
492 } else {
493 const uint ca = const_alpha | (const_alpha << 8); // adjust to [0-65535]
494 const uint cia = 65535 - ca;
495
496 int x = 0;
497
498 // 1) prologue, align on 32 bytes
499 for (; x < length && (quintptr(dst + x) & 31); ++x)
500 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
501
502 // 2) interpolate pixels with AVX2
503 const __m256i half = _mm256_set1_epi32(0x8000);
504 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
505 const __m256i constAlphaVector = _mm256_set1_epi32(ca);
506 const __m256i oneMinusConstAlpha = _mm256_set1_epi32(cia);
507 for (; x < length - 3; x += 4) {
508 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
509 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
510 INTERPOLATE_PIXEL_RGB64_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
511 _mm256_store_si256((__m256i *)&dst[x], dstVector);
512 }
513
514 // 3) Epilogue
515 SIMD_EPILOGUE(x, length, 3)
516 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
517 }
518}
519#endif
520
521#if QT_CONFIG(raster_fp)
522void QT_FASTCALL comp_func_Source_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
523{
524 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
525 if (const_alpha == 255) {
526 ::memcpy(dst, src, length * sizeof(QRgbaFloat32));
527 } else {
528 const float ca = const_alpha / 255.f;
529 const float cia = 1.0f - ca;
530
531 const __m128 constAlphaVector = _mm_set1_ps(ca);
532 const __m128 oneMinusConstAlpha = _mm_set1_ps(cia);
533 const __m256 constAlphaVector256 = _mm256_set1_ps(ca);
534 const __m256 oneMinusConstAlpha256 = _mm256_set1_ps(cia);
535 int x = 0;
536 for (; x < length - 1; x += 2) {
537 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
538 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
539 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
540 dstVector = _mm256_mul_ps(dstVector, oneMinusConstAlpha256);
541 dstVector = _mm256_add_ps(dstVector, srcVector);
542 _mm256_storeu_ps((float *)&dst[x], dstVector);
543 }
544 if (x < length) {
545 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
546 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
547 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
548 dstVector = _mm_mul_ps(dstVector, oneMinusConstAlpha);
549 dstVector = _mm_add_ps(dstVector, srcVector);
550 _mm_storeu_ps((float *)&dst[x], dstVector);
551 }
552 }
553}
554#endif
555
556void QT_FASTCALL comp_func_solid_SourceOver_avx2(uint *destPixels, int length, uint color, uint const_alpha)
557{
558 if ((const_alpha & qAlpha(color)) == 255) {
559 qt_memfill32(destPixels, color, length);
560 } else {
561 if (const_alpha != 255)
562 color = BYTE_MUL(color, const_alpha);
563
564 const quint32 minusAlphaOfColor = qAlpha(~color);
565 int x = 0;
566
567 quint32 *dst = (quint32 *) destPixels;
568 const __m256i colorVector = _mm256_set1_epi32(color);
569 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
570 const __m256i half = _mm256_set1_epi16(0x80);
571 const __m256i minusAlphaOfColorVector = _mm256_set1_epi16(minusAlphaOfColor);
572
573 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
574 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
575
576 for (; x < length - 7; x += 8) {
577 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
578 BYTE_MUL_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
579 dstVector = _mm256_add_epi8(colorVector, dstVector);
580 _mm256_store_si256((__m256i *)&dst[x], dstVector);
581 }
582 SIMD_EPILOGUE(x, length, 7)
583 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
584 }
585}
586
587#if QT_CONFIG(raster_64bit)
588void QT_FASTCALL comp_func_solid_SourceOver_rgb64_avx2(QRgba64 *destPixels, int length, QRgba64 color, uint const_alpha)
589{
590 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
591 if (const_alpha == 255 && color.isOpaque()) {
592 qt_memfill64((quint64*)destPixels, color, length);
593 } else {
594 if (const_alpha != 255)
595 color = multiplyAlpha255(color, const_alpha);
596
597 const uint minusAlphaOfColor = 65535 - color.alpha();
598 int x = 0;
599 quint64 *dst = (quint64 *) destPixels;
600 const __m256i colorVector = _mm256_set1_epi64x(color);
601 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
602 const __m256i half = _mm256_set1_epi32(0x8000);
603 const __m256i minusAlphaOfColorVector = _mm256_set1_epi32(minusAlphaOfColor);
604
605 for (; x < length && (quintptr(dst + x) & 31); ++x)
606 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
607
608 for (; x < length - 3; x += 4) {
609 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
610 BYTE_MUL_RGB64_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
611 dstVector = _mm256_add_epi16(colorVector, dstVector);
612 _mm256_store_si256((__m256i *)&dst[x], dstVector);
613 }
614 SIMD_EPILOGUE(x, length, 3)
615 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
616 }
617}
618#endif
619
620#if QT_CONFIG(raster_fp)
621void QT_FASTCALL comp_func_solid_Source_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
622{
623 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
624 if (const_alpha == 255) {
625 for (int i = 0; i < length; ++i)
626 dst[i] = color;
627 } else {
628 const float a = const_alpha / 255.0f;
629 const __m128 alphaVector = _mm_set1_ps(a);
630 const __m128 minusAlphaVector = _mm_set1_ps(1.0f - a);
631 __m128 colorVector = _mm_loadu_ps((const float *)&color);
632 colorVector = _mm_mul_ps(colorVector, alphaVector);
633 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
634 const __m256 minusAlphaVector256 = _mm256_set1_ps(1.0f - a);
635 int x = 0;
636 for (; x < length - 1; x += 2) {
637 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
638 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
639 dstVector = _mm256_add_ps(dstVector, colorVector256);
640 _mm256_storeu_ps((float *)&dst[x], dstVector);
641 }
642 if (x < length) {
643 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
644 dstVector = _mm_mul_ps(dstVector, minusAlphaVector);
645 dstVector = _mm_add_ps(dstVector, colorVector);
646 _mm_storeu_ps((float *)&dst[x], dstVector);
647 }
648 }
649}
650
651void QT_FASTCALL comp_func_solid_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
652{
653 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
654 if (const_alpha == 255 && color.a >= 1.0f) {
655 for (int i = 0; i < length; ++i)
656 dst[i] = color;
657 } else {
658 __m128 colorVector = _mm_loadu_ps((const float *)&color);
659 if (const_alpha != 255)
660 colorVector = _mm_mul_ps(colorVector, _mm_set1_ps(const_alpha / 255.f));
661 __m128 minusAlphaOfColorVector =
662 _mm_sub_ps(_mm_set1_ps(1.0f), _mm_permute_ps(colorVector, _MM_SHUFFLE(3, 3, 3, 3)));
663 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
664 const __m256 minusAlphaVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(minusAlphaOfColorVector),
665 minusAlphaOfColorVector, 1);
666 int x = 0;
667 for (; x < length - 1; x += 2) {
668 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
669 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
670 dstVector = _mm256_add_ps(dstVector, colorVector256);
671 _mm256_storeu_ps((float *)&dst[x], dstVector);
672 }
673 if (x < length) {
674 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
675 dstVector = _mm_mul_ps(dstVector, minusAlphaOfColorVector);
676 dstVector = _mm_add_ps(dstVector, colorVector);
677 _mm_storeu_ps((float *)&dst[x], dstVector);
678 }
679 }
680}
681#endif
682
683#define interpolate_4_pixels_16_avx2(tlr1, tlr2, blr1, blr2, distx, disty, colorMask, v_256, b) \
684{
685 /* Correct for later unpack */
686 const __m256i vdistx = _mm256_permute4x64_epi64(distx, _MM_SHUFFLE(3, 1, 2, 0));
687 const __m256i vdisty = _mm256_permute4x64_epi64(disty, _MM_SHUFFLE(3, 1, 2, 0));
688
689 __m256i dxdy = _mm256_mullo_epi16 (vdistx, vdisty);
690 const __m256i distx_ = _mm256_slli_epi16(vdistx, 4);
691 const __m256i disty_ = _mm256_slli_epi16(vdisty, 4);
692 __m256i idxidy = _mm256_add_epi16(dxdy, _mm256_sub_epi16(v_256, _mm256_add_epi16(distx_, disty_)));
693 __m256i dxidy = _mm256_sub_epi16(distx_, dxdy);
694 __m256i idxdy = _mm256_sub_epi16(disty_, dxdy);
695
696 __m256i tlr1AG = _mm256_srli_epi16(tlr1, 8);
697 __m256i tlr1RB = _mm256_and_si256(tlr1, colorMask);
698 __m256i tlr2AG = _mm256_srli_epi16(tlr2, 8);
699 __m256i tlr2RB = _mm256_and_si256(tlr2, colorMask);
700 __m256i blr1AG = _mm256_srli_epi16(blr1, 8);
701 __m256i blr1RB = _mm256_and_si256(blr1, colorMask);
702 __m256i blr2AG = _mm256_srli_epi16(blr2, 8);
703 __m256i blr2RB = _mm256_and_si256(blr2, colorMask);
704
705 __m256i odxidy1 = _mm256_unpacklo_epi32(idxidy, dxidy);
706 __m256i odxidy2 = _mm256_unpackhi_epi32(idxidy, dxidy);
707 tlr1AG = _mm256_mullo_epi16(tlr1AG, odxidy1);
708 tlr1RB = _mm256_mullo_epi16(tlr1RB, odxidy1);
709 tlr2AG = _mm256_mullo_epi16(tlr2AG, odxidy2);
710 tlr2RB = _mm256_mullo_epi16(tlr2RB, odxidy2);
711 __m256i odxdy1 = _mm256_unpacklo_epi32(idxdy, dxdy);
712 __m256i odxdy2 = _mm256_unpackhi_epi32(idxdy, dxdy);
713 blr1AG = _mm256_mullo_epi16(blr1AG, odxdy1);
714 blr1RB = _mm256_mullo_epi16(blr1RB, odxdy1);
715 blr2AG = _mm256_mullo_epi16(blr2AG, odxdy2);
716 blr2RB = _mm256_mullo_epi16(blr2RB, odxdy2);
717
718 /* Add the values, and shift to only keep 8 significant bits per colors */
719 __m256i topAG = _mm256_hadd_epi32(tlr1AG, tlr2AG);
720 __m256i topRB = _mm256_hadd_epi32(tlr1RB, tlr2RB);
721 __m256i botAG = _mm256_hadd_epi32(blr1AG, blr2AG);
722 __m256i botRB = _mm256_hadd_epi32(blr1RB, blr2RB);
723 __m256i rAG = _mm256_add_epi16(topAG, botAG);
724 __m256i rRB = _mm256_add_epi16(topRB, botRB);
725 rRB = _mm256_srli_epi16(rRB, 8);
726 /* Correct for hadd */
727 rAG = _mm256_permute4x64_epi64(rAG, _MM_SHUFFLE(3, 1, 2, 0));
728 rRB = _mm256_permute4x64_epi64(rRB, _MM_SHUFFLE(3, 1, 2, 0));
729 _mm256_storeu_si256((__m256i*)(b), _mm256_blendv_epi8(rAG, rRB, colorMask)); \
730}
731
732inline void fetchTransformedBilinear_pixelBounds(int, int l1, int l2, int &v1, int &v2)
733{
734 if (v1 < l1)
735 v2 = v1 = l1;
736 else if (v1 >= l2)
737 v2 = v1 = l2;
738 else
739 v2 = v1 + 1;
740 Q_ASSERT(v1 >= l1 && v1 <= l2);
741 Q_ASSERT(v2 >= l1 && v2 <= l2);
742}
743
744void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx);
745
746void QT_FASTCALL fetchTransformedBilinearARGB32PM_simple_scale_helper_avx2(uint *b, uint *end, const QTextureData &image,
747 int &fx, int &fy, int fdx, int /*fdy*/)
748{
749 int y1 = (fy >> 16);
750 int y2;
751 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
752 const uint *s1 = (const uint *)image.scanLine(y1);
753 const uint *s2 = (const uint *)image.scanLine(y2);
754
755 const int disty = (fy & 0x0000ffff) >> 8;
756 const int idisty = 256 - disty;
757 const int length = end - b;
758
759 // The intermediate buffer is generated in the positive direction
760 const int adjust = (fdx < 0) ? fdx * length : 0;
761 const int offset = (fx + adjust) >> 16;
762 int x = offset;
763
764 Q_DECL_UNINITIALIZED IntermediateBuffer intermediate;
765 // count is the size used in the intermediate_buffer.
766 int count = (qint64(length) * qAbs(fdx) + FixedScale - 1) / FixedScale + 2;
767 // length is supposed to be <= BufferSize either because data->m11 < 1 or
768 // data->m11 < 2, and any larger buffers split
769 Q_ASSERT(count <= BufferSize + 2);
770 int f = 0;
771 int lim = qMin(count, image.x2 - x);
772 if (x < image.x1) {
773 Q_ASSERT(x < image.x2);
774 uint t = s1[image.x1];
775 uint b = s2[image.x1];
776 quint32 rb = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
777 quint32 ag = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
778 do {
779 intermediate.buffer_rb[f] = rb;
780 intermediate.buffer_ag[f] = ag;
781 f++;
782 x++;
783 } while (x < image.x1 && f < lim);
784 }
785
786 const __m256i disty_ = _mm256_set1_epi16(disty);
787 const __m256i idisty_ = _mm256_set1_epi16(idisty);
788 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
789
790 lim -= 7;
791 for (; f < lim; x += 8, f += 8) {
792 // Load 8 pixels from s1, and split the alpha-green and red-blue component
793 __m256i top = _mm256_loadu_si256((const __m256i*)((const uint *)(s1)+x));
794 __m256i topAG = _mm256_srli_epi16(top, 8);
795 __m256i topRB = _mm256_and_si256(top, colorMask);
796 // Multiplies each color component by idisty
797 topAG = _mm256_mullo_epi16 (topAG, idisty_);
798 topRB = _mm256_mullo_epi16 (topRB, idisty_);
799
800 // Same for the s2 vector
801 __m256i bottom = _mm256_loadu_si256((const __m256i*)((const uint *)(s2)+x));
802 __m256i bottomAG = _mm256_srli_epi16(bottom, 8);
803 __m256i bottomRB = _mm256_and_si256(bottom, colorMask);
804 bottomAG = _mm256_mullo_epi16 (bottomAG, disty_);
805 bottomRB = _mm256_mullo_epi16 (bottomRB, disty_);
806
807 // Add the values, and shift to only keep 8 significant bits per colors
808 __m256i rAG =_mm256_add_epi16(topAG, bottomAG);
809 rAG = _mm256_srli_epi16(rAG, 8);
810 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_ag[f]), rAG);
811 __m256i rRB =_mm256_add_epi16(topRB, bottomRB);
812 rRB = _mm256_srli_epi16(rRB, 8);
813 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_rb[f]), rRB);
814 }
815
816 for (; f < count; f++) { // Same as above but without simd
817 x = qMin(x, image.x2 - 1);
818
819 uint t = s1[x];
820 uint b = s2[x];
821
822 intermediate.buffer_rb[f] = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
823 intermediate.buffer_ag[f] = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
824 x++;
825 }
826
827 // Now interpolate the values from the intermediate_buffer to get the final result.
828 intermediate_adder_avx2(b, end, intermediate, offset, fx, fdx);
829}
830
831void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx)
832{
833 fx -= offset * FixedScale;
834
835 const __m128i v_fdx = _mm_set1_epi32(fdx * 4);
836 const __m128i v_blend = _mm_set1_epi32(0x00800080);
837 const __m128i vdx_shuffle = _mm_set_epi8(char(0x80), 13, char(0x80), 13, char(0x80), 9, char(0x80), 9,
838 char(0x80), 5, char(0x80), 5, char(0x80), 1, char(0x80), 1);
839 __m128i v_fx = _mm_setr_epi32(fx, fx + fdx, fx + fdx + fdx, fx + fdx + fdx + fdx);
840
841 while (b < end - 3) {
842 const __m128i offset = _mm_srli_epi32(v_fx, 16);
843 __m256i vrb = _mm256_i32gather_epi64((const long long *)intermediate.buffer_rb, offset, 4);
844 __m256i vag = _mm256_i32gather_epi64((const long long *)intermediate.buffer_ag, offset, 4);
845
846 __m128i vdx = _mm_shuffle_epi8(v_fx, vdx_shuffle);
847 __m128i vidx = _mm_sub_epi16(_mm_set1_epi16(256), vdx);
848 __m256i vmulx = _mm256_castsi128_si256(_mm_unpacklo_epi32(vidx, vdx));
849 vmulx = _mm256_inserti128_si256(vmulx, _mm_unpackhi_epi32(vidx, vdx), 1);
850
851 vrb = _mm256_mullo_epi16(vrb, vmulx);
852 vag = _mm256_mullo_epi16(vag, vmulx);
853
854 __m256i vrbag = _mm256_hadd_epi32(vrb, vag);
855 vrbag = _mm256_permute4x64_epi64(vrbag, _MM_SHUFFLE(3, 1, 2, 0));
856
857 __m128i rb = _mm256_castsi256_si128(vrbag);
858 __m128i ag = _mm256_extracti128_si256(vrbag, 1);
859 rb = _mm_srli_epi16(rb, 8);
860
861 _mm_storeu_si128((__m128i*)b, _mm_blendv_epi8(ag, rb, v_blend));
862
863 b += 4;
864 v_fx = _mm_add_epi32(v_fx, v_fdx);
865 }
866 fx = _mm_cvtsi128_si32(v_fx);
867 while (b < end) {
868 const int x = (fx >> 16);
869
870 const uint distx = (fx & 0x0000ffff) >> 8;
871 const uint idistx = 256 - distx;
872 const uint rb = (intermediate.buffer_rb[x] * idistx + intermediate.buffer_rb[x + 1] * distx) & 0xff00ff00;
873 const uint ag = (intermediate.buffer_ag[x] * idistx + intermediate.buffer_ag[x + 1] * distx) & 0xff00ff00;
874 *b = (rb >> 8) | ag;
875 b++;
876 fx += fdx;
877 }
878 fx += offset * FixedScale;
879}
880
881void QT_FASTCALL fetchTransformedBilinearARGB32PM_downscale_helper_avx2(uint *b, uint *end, const QTextureData &image,
882 int &fx, int &fy, int fdx, int /*fdy*/)
883{
884 int y1 = (fy >> 16);
885 int y2;
886 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
887 const uint *s1 = (const uint *)image.scanLine(y1);
888 const uint *s2 = (const uint *)image.scanLine(y2);
889 const int disty8 = (fy & 0x0000ffff) >> 8;
890 const int disty4 = (disty8 + 0x08) >> 4;
891
892 const qint64 min_fx = qint64(image.x1) * FixedScale;
893 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
894 while (b < end) {
895 int x1 = (fx >> 16);
896 int x2;
897 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
898 if (x1 != x2)
899 break;
900 uint top = s1[x1];
901 uint bot = s2[x1];
902 *b = INTERPOLATE_PIXEL_256(top, 256 - disty8, bot, disty8);
903 fx += fdx;
904 ++b;
905 }
906 uint *boundedEnd = end;
907 if (fdx > 0)
908 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
909 else if (fdx < 0)
910 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
911
912 // A fast middle part without boundary checks
913 const __m256i vdistShuffle =
914 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
915 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
916 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
917 const __m256i v_256 = _mm256_set1_epi16(256);
918 const __m256i v_disty = _mm256_set1_epi16(disty4);
919 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
920 const __m256i v_fx_r = _mm256_set1_epi32(0x08);
921 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
922 __m256i v_fx = _mm256_set1_epi32(fx);
923 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
924
925 while (b < boundedEnd - 7) {
926 const __m256i offset = _mm256_srli_epi32(v_fx, 16);
927 const __m128i offsetLo = _mm256_castsi256_si128(offset);
928 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
929 const __m256i toplo = _mm256_i32gather_epi64((const long long *)s1, offsetLo, 4);
930 const __m256i tophi = _mm256_i32gather_epi64((const long long *)s1, offsetHi, 4);
931 const __m256i botlo = _mm256_i32gather_epi64((const long long *)s2, offsetLo, 4);
932 const __m256i bothi = _mm256_i32gather_epi64((const long long *)s2, offsetHi, 4);
933
934 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
935 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fx_r), 4);
936 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
937
938 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
939 b += 8;
940 v_fx = _mm256_add_epi32(v_fx, v_fdx);
941 }
942 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
943
944 while (b < boundedEnd) {
945 int x = (fx >> 16);
946 int distx8 = (fx & 0x0000ffff) >> 8;
947 *b = interpolate_4_pixels(s1 + x, s2 + x, distx8, disty8);
948 fx += fdx;
949 ++b;
950 }
951
952 while (b < end) {
953 int x1 = (fx >> 16);
954 int x2;
955 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
956 uint tl = s1[x1];
957 uint tr = s1[x2];
958 uint bl = s2[x1];
959 uint br = s2[x2];
960 int distx8 = (fx & 0x0000ffff) >> 8;
961 *b = interpolate_4_pixels(tl, tr, bl, br, distx8, disty8);
962 fx += fdx;
963 ++b;
964 }
965}
966
967void QT_FASTCALL fetchTransformedBilinearARGB32PM_fast_rotate_helper_avx2(uint *b, uint *end, const QTextureData &image,
968 int &fx, int &fy, int fdx, int fdy)
969{
970 const qint64 min_fx = qint64(image.x1) * FixedScale;
971 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
972 const qint64 min_fy = qint64(image.y1) * FixedScale;
973 const qint64 max_fy = qint64(image.y2 - 1) * FixedScale;
974 // first handle the possibly bounded part in the beginning
975 while (b < end) {
976 int x1 = (fx >> 16);
977 int x2;
978 int y1 = (fy >> 16);
979 int y2;
980 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
981 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
982 if (x1 != x2 && y1 != y2)
983 break;
984 const uint *s1 = (const uint *)image.scanLine(y1);
985 const uint *s2 = (const uint *)image.scanLine(y2);
986 uint tl = s1[x1];
987 uint tr = s1[x2];
988 uint bl = s2[x1];
989 uint br = s2[x2];
990 int distx = (fx & 0x0000ffff) >> 8;
991 int disty = (fy & 0x0000ffff) >> 8;
992 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
993 fx += fdx;
994 fy += fdy;
995 ++b;
996 }
997 uint *boundedEnd = end;
998 if (fdx > 0)
999 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
1000 else if (fdx < 0)
1001 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
1002 if (fdy > 0)
1003 boundedEnd = qMin(boundedEnd, b + (max_fy - fy) / fdy);
1004 else if (fdy < 0)
1005 boundedEnd = qMin(boundedEnd, b + (min_fy - fy) / fdy);
1006
1007 // until boundedEnd we can now have a fast middle part without boundary checks
1008 const __m256i vdistShuffle =
1009 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
1010 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
1011 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
1012 const __m256i v_256 = _mm256_set1_epi16(256);
1013 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
1014 const __m256i v_fdy = _mm256_set1_epi32(fdy * 8);
1015 const __m256i v_fxy_r = _mm256_set1_epi32(0x08);
1016 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1017 __m256i v_fx = _mm256_set1_epi32(fx);
1018 __m256i v_fy = _mm256_set1_epi32(fy);
1019 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
1020 v_fy = _mm256_add_epi32(v_fy, _mm256_mullo_epi32(_mm256_set1_epi32(fdy), v_index));
1021
1022 const uchar *textureData = image.imageData;
1023 const qsizetype bytesPerLine = image.bytesPerLine;
1024 const __m256i vbpl = _mm256_set1_epi16(bytesPerLine/4);
1025
1026 while (b < boundedEnd - 7) {
1027 const __m256i vy = _mm256_packs_epi32(_mm256_srli_epi32(v_fy, 16), _mm256_setzero_si256());
1028 // 8x16bit * 8x16bit -> 8x32bit
1029 __m256i offset = _mm256_unpacklo_epi16(_mm256_mullo_epi16(vy, vbpl), _mm256_mulhi_epi16(vy, vbpl));
1030 offset = _mm256_add_epi32(offset, _mm256_srli_epi32(v_fx, 16));
1031 const __m128i offsetLo = _mm256_castsi256_si128(offset);
1032 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
1033 const uint *topData = (const uint *)(textureData);
1034 const uint *botData = (const uint *)(textureData + bytesPerLine);
1035 const __m256i toplo = _mm256_i32gather_epi64((const long long *)topData, offsetLo, 4);
1036 const __m256i tophi = _mm256_i32gather_epi64((const long long *)topData, offsetHi, 4);
1037 const __m256i botlo = _mm256_i32gather_epi64((const long long *)botData, offsetLo, 4);
1038 const __m256i bothi = _mm256_i32gather_epi64((const long long *)botData, offsetHi, 4);
1039
1040 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
1041 __m256i v_disty = _mm256_srli_epi16(v_fy, 8);
1042 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fxy_r), 4);
1043 v_disty = _mm256_srli_epi16(_mm256_add_epi32(v_disty, v_fxy_r), 4);
1044 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
1045 v_disty = _mm256_shuffle_epi8(v_disty, vdistShuffle);
1046
1047 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
1048 b += 8;
1049 v_fx = _mm256_add_epi32(v_fx, v_fdx);
1050 v_fy = _mm256_add_epi32(v_fy, v_fdy);
1051 }
1052 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
1053 fy = _mm_extract_epi32(_mm256_castsi256_si128(v_fy) , 0);
1054
1055 while (b < boundedEnd) {
1056 int x = (fx >> 16);
1057 int y = (fy >> 16);
1058
1059 const uint *s1 = (const uint *)image.scanLine(y);
1060 const uint *s2 = (const uint *)image.scanLine(y + 1);
1061
1062 int distx = (fx & 0x0000ffff) >> 8;
1063 int disty = (fy & 0x0000ffff) >> 8;
1064 *b = interpolate_4_pixels(s1 + x, s2 + x, distx, disty);
1065
1066 fx += fdx;
1067 fy += fdy;
1068 ++b;
1069 }
1070
1071 while (b < end) {
1072 int x1 = (fx >> 16);
1073 int x2;
1074 int y1 = (fy >> 16);
1075 int y2;
1076
1077 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
1078 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
1079
1080 const uint *s1 = (const uint *)image.scanLine(y1);
1081 const uint *s2 = (const uint *)image.scanLine(y2);
1082
1083 uint tl = s1[x1];
1084 uint tr = s1[x2];
1085 uint bl = s2[x1];
1086 uint br = s2[x2];
1087
1088 int distx = (fx & 0x0000ffff) >> 8;
1089 int disty = (fy & 0x0000ffff) >> 8;
1090 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
1091
1092 fx += fdx;
1093 fy += fdy;
1094 ++b;
1095 }
1096}
1097
1098static inline __m256i epilogueMaskFromCount(qsizetype count)
1099{
1100 Q_ASSERT(count > 0);
1101 static const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1102 return _mm256_add_epi32(offsetMask, _mm256_set1_epi32(-count));
1103}
1104
1105template<bool RGBA>
1106static void convertARGBToARGB32PM_avx2(uint *buffer, const uint *src, qsizetype count)
1107{
1108 qsizetype i = 0;
1109 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1110 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1111 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1112 const __m256i half = _mm256_set1_epi16(0x0080);
1113 const __m256i zero = _mm256_setzero_si256();
1114
1115 for (; i < count - 7; i += 8) {
1116 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1117 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1118 // keep the two _mm_test[zc]_siXXX next to each other
1119 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1120 if (RGBA)
1121 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1122 if (!cf) {
1123 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1124 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1125 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1126 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1127 src1 = _mm256_mullo_epi16(src1, alpha1);
1128 src2 = _mm256_mullo_epi16(src2, alpha2);
1129 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1130 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1131 src1 = _mm256_add_epi16(src1, half);
1132 src2 = _mm256_add_epi16(src2, half);
1133 src1 = _mm256_srli_epi16(src1, 8);
1134 src2 = _mm256_srli_epi16(src2, 8);
1135 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1136 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1137 srcVector = _mm256_packus_epi16(src1, src2);
1138 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1139 } else {
1140 if (buffer != src || RGBA)
1141 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1142 }
1143 } else {
1144 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), zero);
1145 }
1146 }
1147
1148 if (i < count) {
1149 const __m256i epilogueMask = epilogueMaskFromCount(count - i);
1150 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1151 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1152
1153 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1154 // keep the two _mm_test[zc]_siXXX next to each other
1155 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1156 if (RGBA)
1157 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1158 if (!cf) {
1159 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1160 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1161 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1162 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1163 src1 = _mm256_mullo_epi16(src1, alpha1);
1164 src2 = _mm256_mullo_epi16(src2, alpha2);
1165 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1166 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1167 src1 = _mm256_add_epi16(src1, half);
1168 src2 = _mm256_add_epi16(src2, half);
1169 src1 = _mm256_srli_epi16(src1, 8);
1170 src2 = _mm256_srli_epi16(src2, 8);
1171 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1172 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1173 srcVector = _mm256_packus_epi16(src1, src2);
1174 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1175 } else {
1176 if (buffer != src || RGBA)
1177 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1178 }
1179 } else {
1180 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, zero);
1181 }
1182 }
1183}
1184
1185void QT_FASTCALL convertARGB32ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1186{
1187 convertARGBToARGB32PM_avx2<false>(buffer, buffer, count);
1188}
1189
1190void QT_FASTCALL convertRGBA8888ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1191{
1192 convertARGBToARGB32PM_avx2<true>(buffer, buffer, count);
1193}
1194
1195const uint *QT_FASTCALL fetchARGB32ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1196 const QList<QRgb> *, QDitherInfo *)
1197{
1198 convertARGBToARGB32PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1199 return buffer;
1200}
1201
1202const uint *QT_FASTCALL fetchRGBA8888ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1203 const QList<QRgb> *, QDitherInfo *)
1204{
1205 convertARGBToARGB32PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1206 return buffer;
1207}
1208
1209template<bool RGBA>
1210static void convertARGBToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, qsizetype count)
1211{
1212 qsizetype i = 0;
1213 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1214 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1215 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1216 const __m256i zero = _mm256_setzero_si256();
1217
1218 for (; i < count - 7; i += 8) {
1219 __m256i dst1, dst2;
1220 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1221 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1222 // keep the two _mm_test[zc]_siXXX next to each other
1223 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1224 if (!RGBA)
1225 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1226
1227 // The two unpack instructions unpack the low and upper halves of
1228 // each 128-bit half of the 256-bit register. Here's the tracking
1229 // of what's where: (p is 32-bit, P is 64-bit)
1230 // as loaded: [ p1, p2, p3, p4; p5, p6, p7, p8 ]
1231 // after permute4x64 [ p1, p2, p5, p6; p3, p4, p7, p8 ]
1232 // after unpacklo/hi [ P1, P2; P3, P4 ] [ P5, P6; P7, P8 ]
1233 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1234
1235 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1236 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1237 if (!cf) {
1238 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1239 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1240 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1241 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1242 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1243 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1244 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1245 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1246 } else {
1247 dst1 = src1;
1248 dst2 = src2;
1249 }
1250 } else {
1251 dst1 = dst2 = zero;
1252 }
1253 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), dst1);
1254 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i) + 1, dst2);
1255 }
1256
1257 if (i < count) {
1258 __m256i epilogueMask = epilogueMaskFromCount(count - i);
1259 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1260 __m256i dst1, dst2;
1261 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1262
1263 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1264 // keep the two _mm_test[zc]_siXXX next to each other
1265 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1266 if (!RGBA)
1267 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1268 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1269 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1270 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1271 if (!cf) {
1272 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1273 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1274 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1275 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1276 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1277 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1278 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1279 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1280 } else {
1281 dst1 = src1;
1282 dst2 = src2;
1283 }
1284 } else {
1285 dst1 = dst2 = zero;
1286 }
1287 epilogueMask = _mm256_permute4x64_epi64(epilogueMask, _MM_SHUFFLE(3, 1, 2, 0));
1288 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i),
1289 _mm256_unpacklo_epi32(epilogueMask, epilogueMask),
1290 dst1);
1291 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i + 4),
1292 _mm256_unpackhi_epi32(epilogueMask, epilogueMask),
1293 dst2);
1294 }
1295}
1296
1297const QRgba64 * QT_FASTCALL convertARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1298 const QList<QRgb> *, QDitherInfo *)
1299{
1300 convertARGBToRGBA64PM_avx2<false>(buffer, src, count);
1301 return buffer;
1302}
1303
1304const QRgba64 * QT_FASTCALL convertRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1305 const QList<QRgb> *, QDitherInfo *)
1306{
1307 convertARGBToRGBA64PM_avx2<true>(buffer, src, count);
1308 return buffer;
1309}
1310
1311const QRgba64 *QT_FASTCALL fetchARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1312 const QList<QRgb> *, QDitherInfo *)
1313{
1314 convertARGBToRGBA64PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1315 return buffer;
1316}
1317
1318const QRgba64 *QT_FASTCALL fetchRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1319 const QList<QRgb> *, QDitherInfo *)
1320{
1321 convertARGBToRGBA64PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1322 return buffer;
1323}
1324
1325const QRgba64 *QT_FASTCALL fetchRGBA64ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1326 const QList<QRgb> *, QDitherInfo *)
1327{
1328 const QRgba64 *s = reinterpret_cast<const QRgba64 *>(src) + index;
1329 int i = 0;
1330 const __m256i vh = _mm256_set1_epi32(0x8000);
1331 for (; i < count - 3; i += 4) {
1332 __m256i vs256 = _mm256_loadu_si256((const __m256i *)(s + i));
1333 __m256i va256 = _mm256_shufflelo_epi16(vs256, _MM_SHUFFLE(3, 3, 3, 3));
1334 va256 = _mm256_shufflehi_epi16(va256, _MM_SHUFFLE(3, 3, 3, 3));
1335 const __m256i vmullo = _mm256_mullo_epi16(vs256, va256);
1336 const __m256i vmulhi = _mm256_mulhi_epu16(vs256, va256);
1337 __m256i vslo = _mm256_unpacklo_epi16(vmullo, vmulhi);
1338 __m256i vshi = _mm256_unpackhi_epi16(vmullo, vmulhi);
1339 vslo = _mm256_add_epi32(vslo, _mm256_srli_epi32(vslo, 16));
1340 vshi = _mm256_add_epi32(vshi, _mm256_srli_epi32(vshi, 16));
1341 vslo = _mm256_add_epi32(vslo, vh);
1342 vshi = _mm256_add_epi32(vshi, vh);
1343 vslo = _mm256_srli_epi32(vslo, 16);
1344 vshi = _mm256_srli_epi32(vshi, 16);
1345 vs256 = _mm256_packus_epi32(vslo, vshi);
1346 vs256 = _mm256_blend_epi16(vs256, va256, 0x88);
1347 _mm256_storeu_si256((__m256i *)(buffer + i), vs256);
1348 }
1349 for (; i < count; ++i) {
1350 const auto a = s[i].alpha();
1351 __m128i vs = _mm_loadl_epi64((const __m128i *)(s + i));
1352 __m128i va = _mm_shufflelo_epi16(vs, _MM_SHUFFLE(3, 3, 3, 3));
1353 vs = multiplyAlpha65535(vs, va);
1354 _mm_storel_epi64((__m128i *)(buffer + i), vs);
1355 buffer[i].setAlpha(a);
1356 }
1357 return buffer;
1358}
1359
1360const uint *QT_FASTCALL fetchRGB16FToRGB32_avx2(uint *buffer, const uchar *src, int index, int count,
1361 const QList<QRgb> *, QDitherInfo *)
1362{
1363 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1364 const __m256 vf = _mm256_set1_ps(255.0f);
1365 const __m256 vh = _mm256_set1_ps(0.5f);
1366 int i = 0;
1367 for (; i + 1 < count; i += 2) {
1368 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1369 vsf = _mm256_mul_ps(vsf, vf);
1370 vsf = _mm256_add_ps(vsf, vh);
1371 __m256i vsi = _mm256_cvttps_epi32(vsf);
1372 vsi = _mm256_packs_epi32(vsi, vsi);
1373 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1374 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1375 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1376 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1377 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1378 }
1379 if (i < count) {
1380 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1381 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1382 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1383 __m128i vsi = _mm_cvttps_epi32(vsf);
1384 vsi = _mm_packs_epi32(vsi, vsi);
1385 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1386 vsi = _mm_packus_epi16(vsi, vsi);
1387 buffer[i] = _mm_cvtsi128_si32(vsi);
1388 }
1389 return buffer;
1390}
1391
1392const uint *QT_FASTCALL fetchRGBA16FToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1393 const QList<QRgb> *, QDitherInfo *)
1394{
1395 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1396 const __m256 vf = _mm256_set1_ps(255.0f);
1397 const __m256 vh = _mm256_set1_ps(0.5f);
1398 int i = 0;
1399 for (; i + 1 < count; i += 2) {
1400 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1401 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1402 vsf = _mm256_mul_ps(vsf, vsa);
1403 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1404 vsf = _mm256_mul_ps(vsf, vf);
1405 vsf = _mm256_add_ps(vsf, vh);
1406 __m256i vsi = _mm256_cvttps_epi32(vsf);
1407 vsi = _mm256_packus_epi32(vsi, vsi);
1408 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1409 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1410 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1411 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1412 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1413 }
1414 if (i < count) {
1415 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1416 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1417 vsf = _mm_mul_ps(vsf, vsa);
1418 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1419 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1420 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1421 __m128i vsi = _mm_cvttps_epi32(vsf);
1422 vsi = _mm_packus_epi32(vsi, vsi);
1423 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1424 vsi = _mm_packus_epi16(vsi, vsi);
1425 buffer[i] = _mm_cvtsi128_si32(vsi);
1426 }
1427 return buffer;
1428}
1429
1430const QRgba64 *QT_FASTCALL fetchRGBA16FPMToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1431 const QList<QRgb> *, QDitherInfo *)
1432{
1433 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1434 const __m256 vf = _mm256_set1_ps(65535.0f);
1435 const __m256 vh = _mm256_set1_ps(0.5f);
1436 int i = 0;
1437 for (; i + 1 < count; i += 2) {
1438 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1439 vsf = _mm256_mul_ps(vsf, vf);
1440 vsf = _mm256_add_ps(vsf, vh);
1441 __m256i vsi = _mm256_cvttps_epi32(vsf);
1442 vsi = _mm256_packus_epi32(vsi, vsi);
1443 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1444 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1445 }
1446 if (i < count) {
1447 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1448 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1449 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1450 __m128i vsi = _mm_cvttps_epi32(vsf);
1451 vsi = _mm_packus_epi32(vsi, vsi);
1452 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1453 }
1454 return buffer;
1455}
1456
1457const QRgba64 *QT_FASTCALL fetchRGBA16FToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1458 const QList<QRgb> *, QDitherInfo *)
1459{
1460 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1461 const __m256 vf = _mm256_set1_ps(65535.0f);
1462 const __m256 vh = _mm256_set1_ps(0.5f);
1463 int i = 0;
1464 for (; i + 1 < count; i += 2) {
1465 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1466 __m256 vsa = _mm256_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1467 vsf = _mm256_mul_ps(vsf, vsa);
1468 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1469 vsf = _mm256_mul_ps(vsf, vf);
1470 vsf = _mm256_add_ps(vsf, vh);
1471 __m256i vsi = _mm256_cvttps_epi32(vsf);
1472 vsi = _mm256_packus_epi32(vsi, vsi);
1473 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1474 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1475 }
1476 if (i < count) {
1477 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1478 __m128 vsa = _mm_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1479 vsf = _mm_mul_ps(vsf, vsa);
1480 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1481 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1482 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1483 __m128i vsi = _mm_cvttps_epi32(vsf);
1484 vsi = _mm_packus_epi32(vsi, vsi);
1485 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1486 }
1487 return buffer;
1488}
1489
1490void QT_FASTCALL storeRGB16FFromRGB32_avx2(uchar *dest, const uint *src, int index, int count,
1491 const QList<QRgb> *, QDitherInfo *)
1492{
1493 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1494 const __m256 vf = _mm256_set1_ps(1.0f / 255.0f);
1495 int i = 0;
1496 for (; i + 1 < count; i += 2) {
1497 __m256i vsi = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(src + i)));
1498 vsi = _mm256_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1499 __m256 vsf = _mm256_cvtepi32_ps(vsi);
1500 vsf = _mm256_mul_ps(vsf, vf);
1501 _mm_storeu_si128((__m128i *)(d + i), _mm256_cvtps_ph(vsf, 0));
1502 }
1503 if (i < count) {
1504 __m128i vsi = _mm_cvtsi32_si128(src[i]);
1505 vsi = _mm_cvtepu8_epi32(vsi);
1506 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1507 __m128 vsf = _mm_cvtepi32_ps(vsi);
1508 vsf = _mm_mul_ps(vsf, _mm_set1_ps(1.0f / 255.0f));
1509 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1510 }
1511}
1512
1513void QT_FASTCALL storeRGBA16FFromARGB32PM_avx2(uchar *dest, const uint *src, int index, int count,
1514 const QList<QRgb> *, QDitherInfo *)
1515{
1516 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1517 const __m128 vf = _mm_set1_ps(1.0f / 255.0f);
1518 for (int i = 0; i < count; ++i) {
1519 const uint s = src[i];
1520 __m128i vsi = _mm_cvtsi32_si128(s);
1521 vsi = _mm_cvtepu8_epi32(vsi);
1522 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1523 __m128 vsf = _mm_cvtepi32_ps(vsi);
1524 const uint8_t a = (s >> 24);
1525 if (a == 255)
1526 vsf = _mm_mul_ps(vsf, vf);
1527 else if (a == 0)
1528 vsf = _mm_set1_ps(0.0f);
1529 else {
1530 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1531 __m128 vsr = _mm_rcp_ps(vsa);
1532 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1533 vsr = _mm_insert_ps(vsr, vf, 0x30);
1534 vsf = _mm_mul_ps(vsf, vsr);
1535 }
1536 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1537 }
1538}
1539
1540#if QT_CONFIG(raster_fp)
1541const QRgbaFloat32 *QT_FASTCALL fetchRGBA16FToRGBA32F_avx2(QRgbaFloat32 *buffer, const uchar *src, int index, int count,
1542 const QList<QRgb> *, QDitherInfo *)
1543{
1544 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1545 int i = 0;
1546 for (; i + 1 < count; i += 2) {
1547 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1548 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1549 vsf = _mm256_mul_ps(vsf, vsa);
1550 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1551 _mm256_storeu_ps((float *)(buffer + i), vsf);
1552 }
1553 if (i < count) {
1554 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1555 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1556 vsf = _mm_mul_ps(vsf, vsa);
1557 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1558 _mm_storeu_ps((float *)(buffer + i), vsf);
1559 }
1560 return buffer;
1561}
1562
1563void QT_FASTCALL storeRGBX16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1564 const QList<QRgb> *, QDitherInfo *)
1565{
1566 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1567 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1568 const __m128 zero = _mm_set_ps(1.0f, 0.0f, 0.0f, 0.0f);
1569 for (int i = 0; i < count; ++i) {
1570 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1571 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1572 const float a = _mm_cvtss_f32(vsa);
1573 if (a == 1.0f)
1574 { }
1575 else if (a == 0.0f)
1576 vsf = zero;
1577 else {
1578 __m128 vsr = _mm_rcp_ps(vsa);
1579 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1580 vsf = _mm_mul_ps(vsf, vsr);
1581 vsf = _mm_insert_ps(vsf, _mm_set_ss(1.0f), 0x30);
1582 }
1583 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1584 }
1585}
1586
1587void QT_FASTCALL storeRGBA16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1588 const QList<QRgb> *, QDitherInfo *)
1589{
1590 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1591 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1592 const __m128 zero = _mm_set1_ps(0.0f);
1593 for (int i = 0; i < count; ++i) {
1594 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1595 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1596 const float a = _mm_cvtss_f32(vsa);
1597 if (a == 1.0f)
1598 { }
1599 else if (a == 0.0f)
1600 vsf = zero;
1601 else {
1602 __m128 vsr = _mm_rcp_ps(vsa);
1603 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1604 vsr = _mm_insert_ps(vsr, _mm_set_ss(1.0f), 0x30);
1605 vsf = _mm_mul_ps(vsf, vsr);
1606 }
1607 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1608 }
1609}
1610#endif
1611
1612QT_END_NAMESPACE
1613
1614#endif