Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qdrawhelper_avx2.cpp
Go to the documentation of this file.
1// Copyright (C) 2018 The Qt Company Ltd.
2// Copyright (C) 2018 Intel Corporation.
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
4
5#include "qdrawhelper_p.h"
6#include "qdrawhelper_x86_p.h"
7#include "qdrawingprimitive_sse2_p.h"
8#include "qpixellayout_p.h"
9#include "qrgba64_p.h"
10
11#if defined(QT_COMPILER_SUPPORTS_AVX2)
12
13QT_BEGIN_NAMESPACE
14
15enum {
16 FixedScale = 1 << 16,
17 HalfPoint = 1 << 15
18};
19
20// Vectorized blend functions:
21
22// See BYTE_MUL_SSE2 for details.
23inline static void Q_DECL_VECTORCALL
24BYTE_MUL_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
25{
26 __m256i pixelVectorAG = _mm256_srli_epi16(pixelVector, 8);
27 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
28
29 pixelVectorAG = _mm256_mullo_epi16(pixelVectorAG, alphaChannel);
30 pixelVectorRB = _mm256_mullo_epi16(pixelVectorRB, alphaChannel);
31
32 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, _mm256_srli_epi16(pixelVectorRB, 8));
33 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, _mm256_srli_epi16(pixelVectorAG, 8));
34 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, half);
35 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, half);
36
37 pixelVectorRB = _mm256_srli_epi16(pixelVectorRB, 8);
38 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
39}
40
41inline static void Q_DECL_VECTORCALL
42BYTE_MUL_RGB64_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
43{
44 __m256i pixelVectorAG = _mm256_srli_epi32(pixelVector, 16);
45 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
46
47 pixelVectorAG = _mm256_mullo_epi32(pixelVectorAG, alphaChannel);
48 pixelVectorRB = _mm256_mullo_epi32(pixelVectorRB, alphaChannel);
49
50 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, _mm256_srli_epi32(pixelVectorRB, 16));
51 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, _mm256_srli_epi32(pixelVectorAG, 16));
52 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, half);
53 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, half);
54
55 pixelVectorRB = _mm256_srli_epi32(pixelVectorRB, 16);
56 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
57}
58
59// See INTERPOLATE_PIXEL_255_SSE2 for details.
60inline static void Q_DECL_VECTORCALL
61INTERPOLATE_PIXEL_255_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
62{
63 const __m256i srcVectorAG = _mm256_srli_epi16(srcVector, 8);
64 const __m256i dstVectorAG = _mm256_srli_epi16(dstVector, 8);
65 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
66 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
67 const __m256i srcVectorAGalpha = _mm256_mullo_epi16(srcVectorAG, alphaChannel);
68 const __m256i srcVectorRBalpha = _mm256_mullo_epi16(srcVectorRB, alphaChannel);
69 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi16(dstVectorAG, oneMinusAlphaChannel);
70 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi16(dstVectorRB, oneMinusAlphaChannel);
71 __m256i finalAG = _mm256_add_epi16(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
72 __m256i finalRB = _mm256_add_epi16(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
73 finalAG = _mm256_add_epi16(finalAG, _mm256_srli_epi16(finalAG, 8));
74 finalRB = _mm256_add_epi16(finalRB, _mm256_srli_epi16(finalRB, 8));
75 finalAG = _mm256_add_epi16(finalAG, half);
76 finalRB = _mm256_add_epi16(finalRB, half);
77 finalRB = _mm256_srli_epi16(finalRB, 8);
78
79 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
80}
81
82inline static void Q_DECL_VECTORCALL
83INTERPOLATE_PIXEL_RGB64_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
84{
85 const __m256i srcVectorAG = _mm256_srli_epi32(srcVector, 16);
86 const __m256i dstVectorAG = _mm256_srli_epi32(dstVector, 16);
87 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
88 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
89 const __m256i srcVectorAGalpha = _mm256_mullo_epi32(srcVectorAG, alphaChannel);
90 const __m256i srcVectorRBalpha = _mm256_mullo_epi32(srcVectorRB, alphaChannel);
91 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi32(dstVectorAG, oneMinusAlphaChannel);
92 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi32(dstVectorRB, oneMinusAlphaChannel);
93 __m256i finalAG = _mm256_add_epi32(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
94 __m256i finalRB = _mm256_add_epi32(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
95 finalAG = _mm256_add_epi32(finalAG, _mm256_srli_epi32(finalAG, 16));
96 finalRB = _mm256_add_epi32(finalRB, _mm256_srli_epi32(finalRB, 16));
97 finalAG = _mm256_add_epi32(finalAG, half);
98 finalRB = _mm256_add_epi32(finalRB, half);
99 finalRB = _mm256_srli_epi32(finalRB, 16);
100 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
101}
102
103// See BLEND_SOURCE_OVER_ARGB32_SSE2 for details.
104inline static void Q_DECL_VECTORCALL BLEND_SOURCE_OVER_ARGB32_AVX2(quint32 *dst, const quint32 *src, const int length)
105{
106 const __m256i half = _mm256_set1_epi16(0x80);
107 const __m256i one = _mm256_set1_epi16(0xff);
108 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
109 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
110 const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
111 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
112 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
113
114 const int minusOffsetToAlignDstOn32Bytes = (reinterpret_cast<quintptr>(dst) >> 2) & 0x7;
115
116 int x = 0;
117 // Prologue to handle all pixels until dst is 32-byte aligned in one step.
118 if (minusOffsetToAlignDstOn32Bytes != 0 && x < (length - 7)) {
119 const __m256i prologueMask = _mm256_sub_epi32(_mm256_set1_epi32(minusOffsetToAlignDstOn32Bytes - 1), offsetMask);
120 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
121 const __m256i prologueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, prologueMask);
122 if (!_mm256_testz_si256(srcVector, prologueAlphaMask)) {
123 if (_mm256_testc_si256(srcVector, prologueAlphaMask)) {
124 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, srcVector);
125 } else {
126 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
127 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
128 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
129 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
130 dstVector = _mm256_add_epi8(dstVector, srcVector);
131 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, dstVector);
132 }
133 }
134 x += (8 - minusOffsetToAlignDstOn32Bytes);
135 }
136
137 for (; x < (length - 7); x += 8) {
138 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
139 if (!_mm256_testz_si256(srcVector, alphaMask)) {
140 if (_mm256_testc_si256(srcVector, alphaMask)) {
141 _mm256_store_si256((__m256i *)&dst[x], srcVector);
142 } else {
143 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
144 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
145 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
146 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
147 dstVector = _mm256_add_epi8(dstVector, srcVector);
148 _mm256_store_si256((__m256i *)&dst[x], dstVector);
149 }
150 }
151 }
152
153 // Epilogue to handle all remaining pixels in one step.
154 if (x < length) {
155 const __m256i epilogueMask = _mm256_add_epi32(offsetMask, _mm256_set1_epi32(x - length));
156 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x], epilogueMask);
157 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
158 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
159 if (_mm256_testc_si256(srcVector, epilogueAlphaMask)) {
160 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, srcVector);
161 } else {
162 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
163 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
164 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x], epilogueMask);
165 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
166 dstVector = _mm256_add_epi8(dstVector, srcVector);
167 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, dstVector);
168 }
169 }
170 }
171}
172
173
174// See BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_SSE2 for details.
175inline static void Q_DECL_VECTORCALL
176BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(quint32 *dst, const quint32 *src, const int length, const int const_alpha)
177{
178 int x = 0;
179
180 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
181 blend_pixel(dst[x], src[x], const_alpha);
182
183 const __m256i half = _mm256_set1_epi16(0x80);
184 const __m256i one = _mm256_set1_epi16(0xff);
185 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
186 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
187 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
188 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
189 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
190 for (; x < (length - 7); x += 8) {
191 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
192 if (!_mm256_testz_si256(srcVector, alphaMask)) {
193 BYTE_MUL_AVX2(srcVector, constAlphaVector, colorMask, half);
194
195 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
196 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
197 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
198 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
199 dstVector = _mm256_add_epi8(dstVector, srcVector);
200 _mm256_store_si256((__m256i *)&dst[x], dstVector);
201 }
202 }
203 SIMD_EPILOGUE(x, length, 7)
204 blend_pixel(dst[x], src[x], const_alpha);
205}
206
207void qt_blend_argb32_on_argb32_avx2(uchar *destPixels, int dbpl,
208 const uchar *srcPixels, int sbpl,
209 int w, int h,
210 int const_alpha)
211{
212 if (const_alpha == 256) {
213 for (int y = 0; y < h; ++y) {
214 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
215 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
216 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, w);
217 destPixels += dbpl;
218 srcPixels += sbpl;
219 }
220 } else if (const_alpha != 0) {
221 const_alpha = (const_alpha * 255) >> 8;
222 for (int y = 0; y < h; ++y) {
223 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
224 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
225 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, w, const_alpha);
226 destPixels += dbpl;
227 srcPixels += sbpl;
228 }
229 }
230}
231
232void qt_blend_rgb32_on_rgb32_avx2(uchar *destPixels, int dbpl,
233 const uchar *srcPixels, int sbpl,
234 int w, int h,
235 int const_alpha)
236{
237 if (const_alpha == 256) {
238 for (int y = 0; y < h; ++y) {
239 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
240 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
241 ::memcpy(dst, src, w * sizeof(uint));
242 srcPixels += sbpl;
243 destPixels += dbpl;
244 }
245 return;
246 }
247 if (const_alpha == 0)
248 return;
249
250 const __m256i half = _mm256_set1_epi16(0x80);
251 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
252
253 const_alpha = (const_alpha * 255) >> 8;
254 int one_minus_const_alpha = 255 - const_alpha;
255 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
256 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(one_minus_const_alpha);
257 for (int y = 0; y < h; ++y) {
258 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
259 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
260 int x = 0;
261
262 // First, align dest to 32 bytes:
263 ALIGNMENT_PROLOGUE_32BYTES(dst, x, w)
264 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
265
266 // 2) interpolate pixels with AVX2
267 for (; x < (w - 7); x += 8) {
268 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
269 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
270 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
271 _mm256_store_si256((__m256i *)&dst[x], dstVector);
272 }
273
274 // 3) Epilogue
275 SIMD_EPILOGUE(x, w, 7)
276 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
277
278 srcPixels += sbpl;
279 destPixels += dbpl;
280 }
281}
282
283Q_NEVER_INLINE static
284void Q_DECL_VECTORCALL qt_memfillXX_avx2(uchar *dest, __m256i value256, qsizetype bytes)
285{
286 __m128i value128 = _mm256_castsi256_si128(value256);
287
288 // main body
289 __m256i *dst256 = reinterpret_cast<__m256i *>(dest);
290 uchar *end = dest + bytes;
291 while (reinterpret_cast<uchar *>(dst256 + 4) <= end) {
292 _mm256_storeu_si256(dst256 + 0, value256);
293 _mm256_storeu_si256(dst256 + 1, value256);
294 _mm256_storeu_si256(dst256 + 2, value256);
295 _mm256_storeu_si256(dst256 + 3, value256);
296 dst256 += 4;
297 }
298
299 // first epilogue: fewer than 128 bytes / 32 entries
300 bytes = end - reinterpret_cast<uchar *>(dst256);
301 switch (bytes / sizeof(value256)) {
302 case 3: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
303 case 2: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
304 case 1: _mm256_storeu_si256(dst256++, value256);
305 }
306
307 // second epilogue: fewer than 32 bytes
308 __m128i *dst128 = reinterpret_cast<__m128i *>(dst256);
309 if (bytes & sizeof(value128))
310 _mm_storeu_si128(dst128++, value128);
311
312 // third epilogue: fewer than 16 bytes
313 if (bytes & 8)
314 _mm_storel_epi64(reinterpret_cast<__m128i *>(end - 8), value128);
315}
316
317void qt_memfill64_avx2(quint64 *dest, quint64 value, qsizetype count)
318{
319#if defined(Q_CC_GNU) && !defined(Q_CC_CLANG)
320 // work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80820
321 __m128i value64 = _mm_set_epi64x(0, value); // _mm_cvtsi64_si128(value);
322# ifdef Q_PROCESSOR_X86_64
323 asm ("" : "+x" (value64));
324# endif
325 __m256i value256 = _mm256_broadcastq_epi64(value64);
326#else
327 __m256i value256 = _mm256_set1_epi64x(value);
328#endif
329
330 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), value256, count * sizeof(quint64));
331}
332
333void qt_memfill32_avx2(quint32 *dest, quint32 value, qsizetype count)
334{
335 if (count % 2) {
336 // odd number of pixels, round to even
337 *dest++ = value;
338 --count;
339 }
340 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), _mm256_set1_epi32(value), count * sizeof(quint32));
341}
342
343void QT_FASTCALL comp_func_SourceOver_avx2(uint *destPixels, const uint *srcPixels, int length, uint const_alpha)
344{
345 Q_ASSERT(const_alpha < 256);
346
347 const quint32 *src = (const quint32 *) srcPixels;
348 quint32 *dst = (quint32 *) destPixels;
349
350 if (const_alpha == 255)
351 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, length);
352 else
353 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, length, const_alpha);
354}
355
356#if QT_CONFIG(raster_64bit)
357void QT_FASTCALL comp_func_SourceOver_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
358{
359 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
360 const __m256i half = _mm256_set1_epi32(0x8000);
361 const __m256i one = _mm256_set1_epi32(0xffff);
362 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
363 __m256i alphaMask = _mm256_set1_epi32(0xff000000);
364 alphaMask = _mm256_unpacklo_epi8(alphaMask, alphaMask);
365 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6,
366 char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6);
367
368 if (const_alpha == 255) {
369 int x = 0;
370 for (; x < length && (quintptr(dst + x) & 31); ++x)
371 blend_pixel(dst[x], src[x]);
372 for (; x < length - 3; x += 4) {
373 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
374 if (!_mm256_testz_si256(srcVector, alphaMask)) {
375 // Not all transparent
376 if (_mm256_testc_si256(srcVector, alphaMask)) {
377 // All opaque
378 _mm256_store_si256((__m256i *)&dst[x], srcVector);
379 } else {
380 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
381 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
382 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
383 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
384 dstVector = _mm256_add_epi16(dstVector, srcVector);
385 _mm256_store_si256((__m256i *)&dst[x], dstVector);
386 }
387 }
388 }
389 SIMD_EPILOGUE(x, length, 3)
390 blend_pixel(dst[x], src[x]);
391 } else {
392 const __m256i constAlphaVector = _mm256_set1_epi32(const_alpha | (const_alpha << 8));
393 int x = 0;
394 for (; x < length && (quintptr(dst + x) & 31); ++x)
395 blend_pixel(dst[x], src[x], const_alpha);
396 for (; x < length - 3; x += 4) {
397 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
398 if (!_mm256_testz_si256(srcVector, alphaMask)) {
399 // Not all transparent
400 BYTE_MUL_RGB64_AVX2(srcVector, constAlphaVector, colorMask, half);
401
402 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
403 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
404 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
405 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
406 dstVector = _mm256_add_epi16(dstVector, srcVector);
407 _mm256_store_si256((__m256i *)&dst[x], dstVector);
408 }
409 }
410 SIMD_EPILOGUE(x, length, 3)
411 blend_pixel(dst[x], src[x], const_alpha);
412 }
413}
414#endif
415
416#if QT_CONFIG(raster_fp)
417void QT_FASTCALL comp_func_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
418{
419 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
420
421 const float a = const_alpha / 255.0f;
422 const __m128 one = _mm_set1_ps(1.0f);
423 const __m128 constAlphaVector = _mm_set1_ps(a);
424 const __m256 one256 = _mm256_set1_ps(1.0f);
425 const __m256 constAlphaVector256 = _mm256_set1_ps(a);
426 int x = 0;
427 for (; x < length - 1; x += 2) {
428 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
429 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
430 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
431 __m256 alphaChannel = _mm256_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
432 alphaChannel = _mm256_sub_ps(one256, alphaChannel);
433 dstVector = _mm256_mul_ps(dstVector, alphaChannel);
434 dstVector = _mm256_add_ps(dstVector, srcVector);
435 _mm256_storeu_ps((float *)(dst + x), dstVector);
436 }
437 if (x < length) {
438 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
439 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
440 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
441 __m128 alphaChannel = _mm_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
442 alphaChannel = _mm_sub_ps(one, alphaChannel);
443 dstVector = _mm_mul_ps(dstVector, alphaChannel);
444 dstVector = _mm_add_ps(dstVector, srcVector);
445 _mm_storeu_ps((float *)(dst + x), dstVector);
446 }
447}
448#endif
449
450void QT_FASTCALL comp_func_Source_avx2(uint *dst, const uint *src, int length, uint const_alpha)
451{
452 if (const_alpha == 255) {
453 ::memcpy(dst, src, length * sizeof(uint));
454 } else {
455 const int ialpha = 255 - const_alpha;
456
457 int x = 0;
458
459 // 1) prologue, align on 32 bytes
460 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
461 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
462
463 // 2) interpolate pixels with AVX2
464 const __m256i half = _mm256_set1_epi16(0x80);
465 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
466 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
467 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(ialpha);
468 for (; x < length - 7; x += 8) {
469 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
470 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
471 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
472 _mm256_store_si256((__m256i *)&dst[x], dstVector);
473 }
474
475 // 3) Epilogue
476 SIMD_EPILOGUE(x, length, 7)
477 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
478 }
479}
480
481#if QT_CONFIG(raster_64bit)
482void QT_FASTCALL comp_func_Source_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
483{
484 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
485 if (const_alpha == 255) {
486 ::memcpy(dst, src, length * sizeof(QRgba64));
487 } else {
488 const uint ca = const_alpha | (const_alpha << 8); // adjust to [0-65535]
489 const uint cia = 65535 - ca;
490
491 int x = 0;
492
493 // 1) prologue, align on 32 bytes
494 for (; x < length && (quintptr(dst + x) & 31); ++x)
495 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
496
497 // 2) interpolate pixels with AVX2
498 const __m256i half = _mm256_set1_epi32(0x8000);
499 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
500 const __m256i constAlphaVector = _mm256_set1_epi32(ca);
501 const __m256i oneMinusConstAlpha = _mm256_set1_epi32(cia);
502 for (; x < length - 3; x += 4) {
503 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
504 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
505 INTERPOLATE_PIXEL_RGB64_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
506 _mm256_store_si256((__m256i *)&dst[x], dstVector);
507 }
508
509 // 3) Epilogue
510 SIMD_EPILOGUE(x, length, 3)
511 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
512 }
513}
514#endif
515
516#if QT_CONFIG(raster_fp)
517void QT_FASTCALL comp_func_Source_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
518{
519 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
520 if (const_alpha == 255) {
521 ::memcpy(dst, src, length * sizeof(QRgbaFloat32));
522 } else {
523 const float ca = const_alpha / 255.f;
524 const float cia = 1.0f - ca;
525
526 const __m128 constAlphaVector = _mm_set1_ps(ca);
527 const __m128 oneMinusConstAlpha = _mm_set1_ps(cia);
528 const __m256 constAlphaVector256 = _mm256_set1_ps(ca);
529 const __m256 oneMinusConstAlpha256 = _mm256_set1_ps(cia);
530 int x = 0;
531 for (; x < length - 1; x += 2) {
532 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
533 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
534 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
535 dstVector = _mm256_mul_ps(dstVector, oneMinusConstAlpha256);
536 dstVector = _mm256_add_ps(dstVector, srcVector);
537 _mm256_storeu_ps((float *)&dst[x], dstVector);
538 }
539 if (x < length) {
540 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
541 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
542 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
543 dstVector = _mm_mul_ps(dstVector, oneMinusConstAlpha);
544 dstVector = _mm_add_ps(dstVector, srcVector);
545 _mm_storeu_ps((float *)&dst[x], dstVector);
546 }
547 }
548}
549#endif
550
551void QT_FASTCALL comp_func_solid_SourceOver_avx2(uint *destPixels, int length, uint color, uint const_alpha)
552{
553 if ((const_alpha & qAlpha(color)) == 255) {
554 qt_memfill32(destPixels, color, length);
555 } else {
556 if (const_alpha != 255)
557 color = BYTE_MUL(color, const_alpha);
558
559 const quint32 minusAlphaOfColor = qAlpha(~color);
560 int x = 0;
561
562 quint32 *dst = (quint32 *) destPixels;
563 const __m256i colorVector = _mm256_set1_epi32(color);
564 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
565 const __m256i half = _mm256_set1_epi16(0x80);
566 const __m256i minusAlphaOfColorVector = _mm256_set1_epi16(minusAlphaOfColor);
567
568 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
569 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
570
571 for (; x < length - 7; x += 8) {
572 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
573 BYTE_MUL_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
574 dstVector = _mm256_add_epi8(colorVector, dstVector);
575 _mm256_store_si256((__m256i *)&dst[x], dstVector);
576 }
577 SIMD_EPILOGUE(x, length, 7)
578 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
579 }
580}
581
582#if QT_CONFIG(raster_64bit)
583void QT_FASTCALL comp_func_solid_SourceOver_rgb64_avx2(QRgba64 *destPixels, int length, QRgba64 color, uint const_alpha)
584{
585 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
586 if (const_alpha == 255 && color.isOpaque()) {
587 qt_memfill64((quint64*)destPixels, color, length);
588 } else {
589 if (const_alpha != 255)
590 color = multiplyAlpha255(color, const_alpha);
591
592 const uint minusAlphaOfColor = 65535 - color.alpha();
593 int x = 0;
594 quint64 *dst = (quint64 *) destPixels;
595 const __m256i colorVector = _mm256_set1_epi64x(color);
596 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
597 const __m256i half = _mm256_set1_epi32(0x8000);
598 const __m256i minusAlphaOfColorVector = _mm256_set1_epi32(minusAlphaOfColor);
599
600 for (; x < length && (quintptr(dst + x) & 31); ++x)
601 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
602
603 for (; x < length - 3; x += 4) {
604 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
605 BYTE_MUL_RGB64_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
606 dstVector = _mm256_add_epi16(colorVector, dstVector);
607 _mm256_store_si256((__m256i *)&dst[x], dstVector);
608 }
609 SIMD_EPILOGUE(x, length, 3)
610 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
611 }
612}
613#endif
614
615#if QT_CONFIG(raster_fp)
616void QT_FASTCALL comp_func_solid_Source_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
617{
618 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
619 if (const_alpha == 255) {
620 for (int i = 0; i < length; ++i)
621 dst[i] = color;
622 } else {
623 const float a = const_alpha / 255.0f;
624 const __m128 alphaVector = _mm_set1_ps(a);
625 const __m128 minusAlphaVector = _mm_set1_ps(1.0f - a);
626 __m128 colorVector = _mm_loadu_ps((const float *)&color);
627 colorVector = _mm_mul_ps(colorVector, alphaVector);
628 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
629 const __m256 minusAlphaVector256 = _mm256_set1_ps(1.0f - a);
630 int x = 0;
631 for (; x < length - 1; x += 2) {
632 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
633 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
634 dstVector = _mm256_add_ps(dstVector, colorVector256);
635 _mm256_storeu_ps((float *)&dst[x], dstVector);
636 }
637 if (x < length) {
638 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
639 dstVector = _mm_mul_ps(dstVector, minusAlphaVector);
640 dstVector = _mm_add_ps(dstVector, colorVector);
641 _mm_storeu_ps((float *)&dst[x], dstVector);
642 }
643 }
644}
645
646void QT_FASTCALL comp_func_solid_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
647{
648 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
649 if (const_alpha == 255 && color.a >= 1.0f) {
650 for (int i = 0; i < length; ++i)
651 dst[i] = color;
652 } else {
653 __m128 colorVector = _mm_loadu_ps((const float *)&color);
654 if (const_alpha != 255)
655 colorVector = _mm_mul_ps(colorVector, _mm_set1_ps(const_alpha / 255.f));
656 __m128 minusAlphaOfColorVector =
657 _mm_sub_ps(_mm_set1_ps(1.0f), _mm_permute_ps(colorVector, _MM_SHUFFLE(3, 3, 3, 3)));
658 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
659 const __m256 minusAlphaVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(minusAlphaOfColorVector),
660 minusAlphaOfColorVector, 1);
661 int x = 0;
662 for (; x < length - 1; x += 2) {
663 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
664 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
665 dstVector = _mm256_add_ps(dstVector, colorVector256);
666 _mm256_storeu_ps((float *)&dst[x], dstVector);
667 }
668 if (x < length) {
669 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
670 dstVector = _mm_mul_ps(dstVector, minusAlphaOfColorVector);
671 dstVector = _mm_add_ps(dstVector, colorVector);
672 _mm_storeu_ps((float *)&dst[x], dstVector);
673 }
674 }
675}
676#endif
677
678#define interpolate_4_pixels_16_avx2(tlr1, tlr2, blr1, blr2, distx, disty, colorMask, v_256, b) \
679{
680 /* Correct for later unpack */
681 const __m256i vdistx = _mm256_permute4x64_epi64(distx, _MM_SHUFFLE(3, 1, 2, 0));
682 const __m256i vdisty = _mm256_permute4x64_epi64(disty, _MM_SHUFFLE(3, 1, 2, 0));
683
684 __m256i dxdy = _mm256_mullo_epi16 (vdistx, vdisty);
685 const __m256i distx_ = _mm256_slli_epi16(vdistx, 4);
686 const __m256i disty_ = _mm256_slli_epi16(vdisty, 4);
687 __m256i idxidy = _mm256_add_epi16(dxdy, _mm256_sub_epi16(v_256, _mm256_add_epi16(distx_, disty_)));
688 __m256i dxidy = _mm256_sub_epi16(distx_, dxdy);
689 __m256i idxdy = _mm256_sub_epi16(disty_, dxdy);
690
691 __m256i tlr1AG = _mm256_srli_epi16(tlr1, 8);
692 __m256i tlr1RB = _mm256_and_si256(tlr1, colorMask);
693 __m256i tlr2AG = _mm256_srli_epi16(tlr2, 8);
694 __m256i tlr2RB = _mm256_and_si256(tlr2, colorMask);
695 __m256i blr1AG = _mm256_srli_epi16(blr1, 8);
696 __m256i blr1RB = _mm256_and_si256(blr1, colorMask);
697 __m256i blr2AG = _mm256_srli_epi16(blr2, 8);
698 __m256i blr2RB = _mm256_and_si256(blr2, colorMask);
699
700 __m256i odxidy1 = _mm256_unpacklo_epi32(idxidy, dxidy);
701 __m256i odxidy2 = _mm256_unpackhi_epi32(idxidy, dxidy);
702 tlr1AG = _mm256_mullo_epi16(tlr1AG, odxidy1);
703 tlr1RB = _mm256_mullo_epi16(tlr1RB, odxidy1);
704 tlr2AG = _mm256_mullo_epi16(tlr2AG, odxidy2);
705 tlr2RB = _mm256_mullo_epi16(tlr2RB, odxidy2);
706 __m256i odxdy1 = _mm256_unpacklo_epi32(idxdy, dxdy);
707 __m256i odxdy2 = _mm256_unpackhi_epi32(idxdy, dxdy);
708 blr1AG = _mm256_mullo_epi16(blr1AG, odxdy1);
709 blr1RB = _mm256_mullo_epi16(blr1RB, odxdy1);
710 blr2AG = _mm256_mullo_epi16(blr2AG, odxdy2);
711 blr2RB = _mm256_mullo_epi16(blr2RB, odxdy2);
712
713 /* Add the values, and shift to only keep 8 significant bits per colors */
714 __m256i topAG = _mm256_hadd_epi32(tlr1AG, tlr2AG);
715 __m256i topRB = _mm256_hadd_epi32(tlr1RB, tlr2RB);
716 __m256i botAG = _mm256_hadd_epi32(blr1AG, blr2AG);
717 __m256i botRB = _mm256_hadd_epi32(blr1RB, blr2RB);
718 __m256i rAG = _mm256_add_epi16(topAG, botAG);
719 __m256i rRB = _mm256_add_epi16(topRB, botRB);
720 rRB = _mm256_srli_epi16(rRB, 8);
721 /* Correct for hadd */
722 rAG = _mm256_permute4x64_epi64(rAG, _MM_SHUFFLE(3, 1, 2, 0));
723 rRB = _mm256_permute4x64_epi64(rRB, _MM_SHUFFLE(3, 1, 2, 0));
724 _mm256_storeu_si256((__m256i*)(b), _mm256_blendv_epi8(rAG, rRB, colorMask)); \
725}
726
727inline void fetchTransformedBilinear_pixelBounds(int, int l1, int l2, int &v1, int &v2)
728{
729 if (v1 < l1)
730 v2 = v1 = l1;
731 else if (v1 >= l2)
732 v2 = v1 = l2;
733 else
734 v2 = v1 + 1;
735 Q_ASSERT(v1 >= l1 && v1 <= l2);
736 Q_ASSERT(v2 >= l1 && v2 <= l2);
737}
738
739void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx);
740
741void QT_FASTCALL fetchTransformedBilinearARGB32PM_simple_scale_helper_avx2(uint *b, uint *end, const QTextureData &image,
742 int &fx, int &fy, int fdx, int /*fdy*/)
743{
744 int y1 = (fy >> 16);
745 int y2;
746 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
747 const uint *s1 = (const uint *)image.scanLine(y1);
748 const uint *s2 = (const uint *)image.scanLine(y2);
749
750 const int disty = (fy & 0x0000ffff) >> 8;
751 const int idisty = 256 - disty;
752 const int length = end - b;
753
754 // The intermediate buffer is generated in the positive direction
755 const int adjust = (fdx < 0) ? fdx * length : 0;
756 const int offset = (fx + adjust) >> 16;
757 int x = offset;
758
759 Q_DECL_UNINITIALIZED IntermediateBuffer intermediate;
760 // count is the size used in the intermediate_buffer.
761 int count = (qint64(length) * qAbs(fdx) + FixedScale - 1) / FixedScale + 2;
762 // length is supposed to be <= BufferSize either because data->m11 < 1 or
763 // data->m11 < 2, and any larger buffers split
764 Q_ASSERT(count <= BufferSize + 2);
765 int f = 0;
766 int lim = qMin(count, image.x2 - x);
767 if (x < image.x1) {
768 Q_ASSERT(x < image.x2);
769 uint t = s1[image.x1];
770 uint b = s2[image.x1];
771 quint32 rb = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
772 quint32 ag = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
773 do {
774 intermediate.buffer_rb[f] = rb;
775 intermediate.buffer_ag[f] = ag;
776 f++;
777 x++;
778 } while (x < image.x1 && f < lim);
779 }
780
781 const __m256i disty_ = _mm256_set1_epi16(disty);
782 const __m256i idisty_ = _mm256_set1_epi16(idisty);
783 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
784
785 lim -= 7;
786 for (; f < lim; x += 8, f += 8) {
787 // Load 8 pixels from s1, and split the alpha-green and red-blue component
788 __m256i top = _mm256_loadu_si256((const __m256i*)((const uint *)(s1)+x));
789 __m256i topAG = _mm256_srli_epi16(top, 8);
790 __m256i topRB = _mm256_and_si256(top, colorMask);
791 // Multiplies each color component by idisty
792 topAG = _mm256_mullo_epi16 (topAG, idisty_);
793 topRB = _mm256_mullo_epi16 (topRB, idisty_);
794
795 // Same for the s2 vector
796 __m256i bottom = _mm256_loadu_si256((const __m256i*)((const uint *)(s2)+x));
797 __m256i bottomAG = _mm256_srli_epi16(bottom, 8);
798 __m256i bottomRB = _mm256_and_si256(bottom, colorMask);
799 bottomAG = _mm256_mullo_epi16 (bottomAG, disty_);
800 bottomRB = _mm256_mullo_epi16 (bottomRB, disty_);
801
802 // Add the values, and shift to only keep 8 significant bits per colors
803 __m256i rAG =_mm256_add_epi16(topAG, bottomAG);
804 rAG = _mm256_srli_epi16(rAG, 8);
805 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_ag[f]), rAG);
806 __m256i rRB =_mm256_add_epi16(topRB, bottomRB);
807 rRB = _mm256_srli_epi16(rRB, 8);
808 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_rb[f]), rRB);
809 }
810
811 for (; f < count; f++) { // Same as above but without simd
812 x = qMin(x, image.x2 - 1);
813
814 uint t = s1[x];
815 uint b = s2[x];
816
817 intermediate.buffer_rb[f] = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
818 intermediate.buffer_ag[f] = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
819 x++;
820 }
821
822 // Now interpolate the values from the intermediate_buffer to get the final result.
823 intermediate_adder_avx2(b, end, intermediate, offset, fx, fdx);
824}
825
826void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx)
827{
828 fx -= offset * FixedScale;
829
830 const __m128i v_fdx = _mm_set1_epi32(fdx * 4);
831 const __m128i v_blend = _mm_set1_epi32(0x00800080);
832 const __m128i vdx_shuffle = _mm_set_epi8(char(0x80), 13, char(0x80), 13, char(0x80), 9, char(0x80), 9,
833 char(0x80), 5, char(0x80), 5, char(0x80), 1, char(0x80), 1);
834 __m128i v_fx = _mm_setr_epi32(fx, fx + fdx, fx + fdx + fdx, fx + fdx + fdx + fdx);
835
836 while (b < end - 3) {
837 const __m128i offset = _mm_srli_epi32(v_fx, 16);
838 __m256i vrb = _mm256_i32gather_epi64((const long long *)intermediate.buffer_rb, offset, 4);
839 __m256i vag = _mm256_i32gather_epi64((const long long *)intermediate.buffer_ag, offset, 4);
840
841 __m128i vdx = _mm_shuffle_epi8(v_fx, vdx_shuffle);
842 __m128i vidx = _mm_sub_epi16(_mm_set1_epi16(256), vdx);
843 __m256i vmulx = _mm256_castsi128_si256(_mm_unpacklo_epi32(vidx, vdx));
844 vmulx = _mm256_inserti128_si256(vmulx, _mm_unpackhi_epi32(vidx, vdx), 1);
845
846 vrb = _mm256_mullo_epi16(vrb, vmulx);
847 vag = _mm256_mullo_epi16(vag, vmulx);
848
849 __m256i vrbag = _mm256_hadd_epi32(vrb, vag);
850 vrbag = _mm256_permute4x64_epi64(vrbag, _MM_SHUFFLE(3, 1, 2, 0));
851
852 __m128i rb = _mm256_castsi256_si128(vrbag);
853 __m128i ag = _mm256_extracti128_si256(vrbag, 1);
854 rb = _mm_srli_epi16(rb, 8);
855
856 _mm_storeu_si128((__m128i*)b, _mm_blendv_epi8(ag, rb, v_blend));
857
858 b += 4;
859 v_fx = _mm_add_epi32(v_fx, v_fdx);
860 }
861 fx = _mm_cvtsi128_si32(v_fx);
862 while (b < end) {
863 const int x = (fx >> 16);
864
865 const uint distx = (fx & 0x0000ffff) >> 8;
866 const uint idistx = 256 - distx;
867 const uint rb = (intermediate.buffer_rb[x] * idistx + intermediate.buffer_rb[x + 1] * distx) & 0xff00ff00;
868 const uint ag = (intermediate.buffer_ag[x] * idistx + intermediate.buffer_ag[x + 1] * distx) & 0xff00ff00;
869 *b = (rb >> 8) | ag;
870 b++;
871 fx += fdx;
872 }
873 fx += offset * FixedScale;
874}
875
876void QT_FASTCALL fetchTransformedBilinearARGB32PM_downscale_helper_avx2(uint *b, uint *end, const QTextureData &image,
877 int &fx, int &fy, int fdx, int /*fdy*/)
878{
879 int y1 = (fy >> 16);
880 int y2;
881 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
882 const uint *s1 = (const uint *)image.scanLine(y1);
883 const uint *s2 = (const uint *)image.scanLine(y2);
884 const int disty8 = (fy & 0x0000ffff) >> 8;
885 const int disty4 = (disty8 + 0x08) >> 4;
886
887 const qint64 min_fx = qint64(image.x1) * FixedScale;
888 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
889 while (b < end) {
890 int x1 = (fx >> 16);
891 int x2;
892 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
893 if (x1 != x2)
894 break;
895 uint top = s1[x1];
896 uint bot = s2[x1];
897 *b = INTERPOLATE_PIXEL_256(top, 256 - disty8, bot, disty8);
898 fx += fdx;
899 ++b;
900 }
901 uint *boundedEnd = end;
902 if (fdx > 0)
903 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
904 else if (fdx < 0)
905 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
906
907 // A fast middle part without boundary checks
908 const __m256i vdistShuffle =
909 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
910 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
911 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
912 const __m256i v_256 = _mm256_set1_epi16(256);
913 const __m256i v_disty = _mm256_set1_epi16(disty4);
914 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
915 const __m256i v_fx_r = _mm256_set1_epi32(0x08);
916 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
917 __m256i v_fx = _mm256_set1_epi32(fx);
918 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
919
920 while (b < boundedEnd - 7) {
921 const __m256i offset = _mm256_srli_epi32(v_fx, 16);
922 const __m128i offsetLo = _mm256_castsi256_si128(offset);
923 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
924 const __m256i toplo = _mm256_i32gather_epi64((const long long *)s1, offsetLo, 4);
925 const __m256i tophi = _mm256_i32gather_epi64((const long long *)s1, offsetHi, 4);
926 const __m256i botlo = _mm256_i32gather_epi64((const long long *)s2, offsetLo, 4);
927 const __m256i bothi = _mm256_i32gather_epi64((const long long *)s2, offsetHi, 4);
928
929 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
930 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fx_r), 4);
931 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
932
933 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
934 b += 8;
935 v_fx = _mm256_add_epi32(v_fx, v_fdx);
936 }
937 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
938
939 while (b < boundedEnd) {
940 int x = (fx >> 16);
941 int distx8 = (fx & 0x0000ffff) >> 8;
942 *b = interpolate_4_pixels(s1 + x, s2 + x, distx8, disty8);
943 fx += fdx;
944 ++b;
945 }
946
947 while (b < end) {
948 int x1 = (fx >> 16);
949 int x2;
950 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
951 uint tl = s1[x1];
952 uint tr = s1[x2];
953 uint bl = s2[x1];
954 uint br = s2[x2];
955 int distx8 = (fx & 0x0000ffff) >> 8;
956 *b = interpolate_4_pixels(tl, tr, bl, br, distx8, disty8);
957 fx += fdx;
958 ++b;
959 }
960}
961
962void QT_FASTCALL fetchTransformedBilinearARGB32PM_fast_rotate_helper_avx2(uint *b, uint *end, const QTextureData &image,
963 int &fx, int &fy, int fdx, int fdy)
964{
965 const qint64 min_fx = qint64(image.x1) * FixedScale;
966 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
967 const qint64 min_fy = qint64(image.y1) * FixedScale;
968 const qint64 max_fy = qint64(image.y2 - 1) * FixedScale;
969 // first handle the possibly bounded part in the beginning
970 while (b < end) {
971 int x1 = (fx >> 16);
972 int x2;
973 int y1 = (fy >> 16);
974 int y2;
975 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
976 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
977 if (x1 != x2 && y1 != y2)
978 break;
979 const uint *s1 = (const uint *)image.scanLine(y1);
980 const uint *s2 = (const uint *)image.scanLine(y2);
981 uint tl = s1[x1];
982 uint tr = s1[x2];
983 uint bl = s2[x1];
984 uint br = s2[x2];
985 int distx = (fx & 0x0000ffff) >> 8;
986 int disty = (fy & 0x0000ffff) >> 8;
987 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
988 fx += fdx;
989 fy += fdy;
990 ++b;
991 }
992 uint *boundedEnd = end;
993 if (fdx > 0)
994 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
995 else if (fdx < 0)
996 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
997 if (fdy > 0)
998 boundedEnd = qMin(boundedEnd, b + (max_fy - fy) / fdy);
999 else if (fdy < 0)
1000 boundedEnd = qMin(boundedEnd, b + (min_fy - fy) / fdy);
1001
1002 // until boundedEnd we can now have a fast middle part without boundary checks
1003 const __m256i vdistShuffle =
1004 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
1005 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
1006 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
1007 const __m256i v_256 = _mm256_set1_epi16(256);
1008 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
1009 const __m256i v_fdy = _mm256_set1_epi32(fdy * 8);
1010 const __m256i v_fxy_r = _mm256_set1_epi32(0x08);
1011 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1012 __m256i v_fx = _mm256_set1_epi32(fx);
1013 __m256i v_fy = _mm256_set1_epi32(fy);
1014 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
1015 v_fy = _mm256_add_epi32(v_fy, _mm256_mullo_epi32(_mm256_set1_epi32(fdy), v_index));
1016
1017 const uchar *textureData = image.imageData;
1018 const qsizetype bytesPerLine = image.bytesPerLine;
1019 const __m256i vbpl = _mm256_set1_epi16(bytesPerLine/4);
1020
1021 while (b < boundedEnd - 7) {
1022 const __m256i vy = _mm256_packs_epi32(_mm256_srli_epi32(v_fy, 16), _mm256_setzero_si256());
1023 // 8x16bit * 8x16bit -> 8x32bit
1024 __m256i offset = _mm256_unpacklo_epi16(_mm256_mullo_epi16(vy, vbpl), _mm256_mulhi_epi16(vy, vbpl));
1025 offset = _mm256_add_epi32(offset, _mm256_srli_epi32(v_fx, 16));
1026 const __m128i offsetLo = _mm256_castsi256_si128(offset);
1027 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
1028 const uint *topData = (const uint *)(textureData);
1029 const uint *botData = (const uint *)(textureData + bytesPerLine);
1030 const __m256i toplo = _mm256_i32gather_epi64((const long long *)topData, offsetLo, 4);
1031 const __m256i tophi = _mm256_i32gather_epi64((const long long *)topData, offsetHi, 4);
1032 const __m256i botlo = _mm256_i32gather_epi64((const long long *)botData, offsetLo, 4);
1033 const __m256i bothi = _mm256_i32gather_epi64((const long long *)botData, offsetHi, 4);
1034
1035 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
1036 __m256i v_disty = _mm256_srli_epi16(v_fy, 8);
1037 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fxy_r), 4);
1038 v_disty = _mm256_srli_epi16(_mm256_add_epi32(v_disty, v_fxy_r), 4);
1039 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
1040 v_disty = _mm256_shuffle_epi8(v_disty, vdistShuffle);
1041
1042 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
1043 b += 8;
1044 v_fx = _mm256_add_epi32(v_fx, v_fdx);
1045 v_fy = _mm256_add_epi32(v_fy, v_fdy);
1046 }
1047 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
1048 fy = _mm_extract_epi32(_mm256_castsi256_si128(v_fy) , 0);
1049
1050 while (b < boundedEnd) {
1051 int x = (fx >> 16);
1052 int y = (fy >> 16);
1053
1054 const uint *s1 = (const uint *)image.scanLine(y);
1055 const uint *s2 = (const uint *)image.scanLine(y + 1);
1056
1057 int distx = (fx & 0x0000ffff) >> 8;
1058 int disty = (fy & 0x0000ffff) >> 8;
1059 *b = interpolate_4_pixels(s1 + x, s2 + x, distx, disty);
1060
1061 fx += fdx;
1062 fy += fdy;
1063 ++b;
1064 }
1065
1066 while (b < end) {
1067 int x1 = (fx >> 16);
1068 int x2;
1069 int y1 = (fy >> 16);
1070 int y2;
1071
1072 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
1073 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
1074
1075 const uint *s1 = (const uint *)image.scanLine(y1);
1076 const uint *s2 = (const uint *)image.scanLine(y2);
1077
1078 uint tl = s1[x1];
1079 uint tr = s1[x2];
1080 uint bl = s2[x1];
1081 uint br = s2[x2];
1082
1083 int distx = (fx & 0x0000ffff) >> 8;
1084 int disty = (fy & 0x0000ffff) >> 8;
1085 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
1086
1087 fx += fdx;
1088 fy += fdy;
1089 ++b;
1090 }
1091}
1092
1093static inline __m256i epilogueMaskFromCount(qsizetype count)
1094{
1095 Q_ASSERT(count > 0);
1096 static const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1097 return _mm256_add_epi32(offsetMask, _mm256_set1_epi32(-count));
1098}
1099
1100template<bool RGBA>
1101static void convertARGBToARGB32PM_avx2(uint *buffer, const uint *src, qsizetype count)
1102{
1103 qsizetype i = 0;
1104 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1105 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1106 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1107 const __m256i half = _mm256_set1_epi16(0x0080);
1108 const __m256i zero = _mm256_setzero_si256();
1109
1110 for (; i < count - 7; i += 8) {
1111 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1112 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1113 // keep the two _mm_test[zc]_siXXX next to each other
1114 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1115 if (RGBA)
1116 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1117 if (!cf) {
1118 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1119 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1120 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1121 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1122 src1 = _mm256_mullo_epi16(src1, alpha1);
1123 src2 = _mm256_mullo_epi16(src2, alpha2);
1124 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1125 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1126 src1 = _mm256_add_epi16(src1, half);
1127 src2 = _mm256_add_epi16(src2, half);
1128 src1 = _mm256_srli_epi16(src1, 8);
1129 src2 = _mm256_srli_epi16(src2, 8);
1130 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1131 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1132 srcVector = _mm256_packus_epi16(src1, src2);
1133 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1134 } else {
1135 if (buffer != src || RGBA)
1136 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1137 }
1138 } else {
1139 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), zero);
1140 }
1141 }
1142
1143 if (i < count) {
1144 const __m256i epilogueMask = epilogueMaskFromCount(count - i);
1145 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1146 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1147
1148 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1149 // keep the two _mm_test[zc]_siXXX next to each other
1150 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1151 if (RGBA)
1152 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1153 if (!cf) {
1154 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1155 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1156 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1157 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1158 src1 = _mm256_mullo_epi16(src1, alpha1);
1159 src2 = _mm256_mullo_epi16(src2, alpha2);
1160 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1161 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1162 src1 = _mm256_add_epi16(src1, half);
1163 src2 = _mm256_add_epi16(src2, half);
1164 src1 = _mm256_srli_epi16(src1, 8);
1165 src2 = _mm256_srli_epi16(src2, 8);
1166 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1167 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1168 srcVector = _mm256_packus_epi16(src1, src2);
1169 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1170 } else {
1171 if (buffer != src || RGBA)
1172 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1173 }
1174 } else {
1175 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, zero);
1176 }
1177 }
1178}
1179
1180void QT_FASTCALL convertARGB32ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1181{
1182 convertARGBToARGB32PM_avx2<false>(buffer, buffer, count);
1183}
1184
1185void QT_FASTCALL convertRGBA8888ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1186{
1187 convertARGBToARGB32PM_avx2<true>(buffer, buffer, count);
1188}
1189
1190const uint *QT_FASTCALL fetchARGB32ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1191 const QList<QRgb> *, QDitherInfo *)
1192{
1193 convertARGBToARGB32PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1194 return buffer;
1195}
1196
1197const uint *QT_FASTCALL fetchRGBA8888ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1198 const QList<QRgb> *, QDitherInfo *)
1199{
1200 convertARGBToARGB32PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1201 return buffer;
1202}
1203
1204template<bool RGBA>
1205static void convertARGBToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, qsizetype count)
1206{
1207 qsizetype i = 0;
1208 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1209 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1210 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1211 const __m256i zero = _mm256_setzero_si256();
1212
1213 for (; i < count - 7; i += 8) {
1214 __m256i dst1, dst2;
1215 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1216 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1217 // keep the two _mm_test[zc]_siXXX next to each other
1218 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1219 if (!RGBA)
1220 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1221
1222 // The two unpack instructions unpack the low and upper halves of
1223 // each 128-bit half of the 256-bit register. Here's the tracking
1224 // of what's where: (p is 32-bit, P is 64-bit)
1225 // as loaded: [ p1, p2, p3, p4; p5, p6, p7, p8 ]
1226 // after permute4x64 [ p1, p2, p5, p6; p3, p4, p7, p8 ]
1227 // after unpacklo/hi [ P1, P2; P3, P4 ] [ P5, P6; P7, P8 ]
1228 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1229
1230 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1231 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1232 if (!cf) {
1233 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1234 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1235 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1236 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1237 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1238 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1239 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1240 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1241 } else {
1242 dst1 = src1;
1243 dst2 = src2;
1244 }
1245 } else {
1246 dst1 = dst2 = zero;
1247 }
1248 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), dst1);
1249 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i) + 1, dst2);
1250 }
1251
1252 if (i < count) {
1253 __m256i epilogueMask = epilogueMaskFromCount(count - i);
1254 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1255 __m256i dst1, dst2;
1256 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1257
1258 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1259 // keep the two _mm_test[zc]_siXXX next to each other
1260 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1261 if (!RGBA)
1262 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1263 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1264 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1265 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1266 if (!cf) {
1267 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1268 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1269 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1270 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1271 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1272 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1273 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1274 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1275 } else {
1276 dst1 = src1;
1277 dst2 = src2;
1278 }
1279 } else {
1280 dst1 = dst2 = zero;
1281 }
1282 epilogueMask = _mm256_permute4x64_epi64(epilogueMask, _MM_SHUFFLE(3, 1, 2, 0));
1283 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i),
1284 _mm256_unpacklo_epi32(epilogueMask, epilogueMask),
1285 dst1);
1286 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i + 4),
1287 _mm256_unpackhi_epi32(epilogueMask, epilogueMask),
1288 dst2);
1289 }
1290}
1291
1292const QRgba64 * QT_FASTCALL convertARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1293 const QList<QRgb> *, QDitherInfo *)
1294{
1295 convertARGBToRGBA64PM_avx2<false>(buffer, src, count);
1296 return buffer;
1297}
1298
1299const QRgba64 * QT_FASTCALL convertRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1300 const QList<QRgb> *, QDitherInfo *)
1301{
1302 convertARGBToRGBA64PM_avx2<true>(buffer, src, count);
1303 return buffer;
1304}
1305
1306const QRgba64 *QT_FASTCALL fetchARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1307 const QList<QRgb> *, QDitherInfo *)
1308{
1309 convertARGBToRGBA64PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1310 return buffer;
1311}
1312
1313const QRgba64 *QT_FASTCALL fetchRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1314 const QList<QRgb> *, QDitherInfo *)
1315{
1316 convertARGBToRGBA64PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1317 return buffer;
1318}
1319
1320const QRgba64 *QT_FASTCALL fetchRGBA64ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1321 const QList<QRgb> *, QDitherInfo *)
1322{
1323 const QRgba64 *s = reinterpret_cast<const QRgba64 *>(src) + index;
1324 int i = 0;
1325 const __m256i vh = _mm256_set1_epi32(0x8000);
1326 for (; i < count - 3; i += 4) {
1327 __m256i vs256 = _mm256_loadu_si256((const __m256i *)(s + i));
1328 __m256i va256 = _mm256_shufflelo_epi16(vs256, _MM_SHUFFLE(3, 3, 3, 3));
1329 va256 = _mm256_shufflehi_epi16(va256, _MM_SHUFFLE(3, 3, 3, 3));
1330 const __m256i vmullo = _mm256_mullo_epi16(vs256, va256);
1331 const __m256i vmulhi = _mm256_mulhi_epu16(vs256, va256);
1332 __m256i vslo = _mm256_unpacklo_epi16(vmullo, vmulhi);
1333 __m256i vshi = _mm256_unpackhi_epi16(vmullo, vmulhi);
1334 vslo = _mm256_add_epi32(vslo, _mm256_srli_epi32(vslo, 16));
1335 vshi = _mm256_add_epi32(vshi, _mm256_srli_epi32(vshi, 16));
1336 vslo = _mm256_add_epi32(vslo, vh);
1337 vshi = _mm256_add_epi32(vshi, vh);
1338 vslo = _mm256_srli_epi32(vslo, 16);
1339 vshi = _mm256_srli_epi32(vshi, 16);
1340 vs256 = _mm256_packus_epi32(vslo, vshi);
1341 vs256 = _mm256_blend_epi16(vs256, va256, 0x88);
1342 _mm256_storeu_si256((__m256i *)(buffer + i), vs256);
1343 }
1344 for (; i < count; ++i) {
1345 const auto a = s[i].alpha();
1346 __m128i vs = _mm_loadl_epi64((const __m128i *)(s + i));
1347 __m128i va = _mm_shufflelo_epi16(vs, _MM_SHUFFLE(3, 3, 3, 3));
1348 vs = multiplyAlpha65535(vs, va);
1349 _mm_storel_epi64((__m128i *)(buffer + i), vs);
1350 buffer[i].setAlpha(a);
1351 }
1352 return buffer;
1353}
1354
1355const uint *QT_FASTCALL fetchRGB16FToRGB32_avx2(uint *buffer, const uchar *src, int index, int count,
1356 const QList<QRgb> *, QDitherInfo *)
1357{
1358 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1359 const __m256 vf = _mm256_set1_ps(255.0f);
1360 const __m256 vh = _mm256_set1_ps(0.5f);
1361 int i = 0;
1362 for (; i + 1 < count; i += 2) {
1363 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1364 vsf = _mm256_mul_ps(vsf, vf);
1365 vsf = _mm256_add_ps(vsf, vh);
1366 __m256i vsi = _mm256_cvttps_epi32(vsf);
1367 vsi = _mm256_packs_epi32(vsi, vsi);
1368 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1369 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1370 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1371 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1372 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1373 }
1374 if (i < count) {
1375 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1376 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1377 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1378 __m128i vsi = _mm_cvttps_epi32(vsf);
1379 vsi = _mm_packs_epi32(vsi, vsi);
1380 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1381 vsi = _mm_packus_epi16(vsi, vsi);
1382 buffer[i] = _mm_cvtsi128_si32(vsi);
1383 }
1384 return buffer;
1385}
1386
1387const uint *QT_FASTCALL fetchRGBA16FToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1388 const QList<QRgb> *, QDitherInfo *)
1389{
1390 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1391 const __m256 vf = _mm256_set1_ps(255.0f);
1392 const __m256 vh = _mm256_set1_ps(0.5f);
1393 int i = 0;
1394 for (; i + 1 < count; i += 2) {
1395 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1396 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1397 vsf = _mm256_mul_ps(vsf, vsa);
1398 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1399 vsf = _mm256_mul_ps(vsf, vf);
1400 vsf = _mm256_add_ps(vsf, vh);
1401 __m256i vsi = _mm256_cvttps_epi32(vsf);
1402 vsi = _mm256_packus_epi32(vsi, vsi);
1403 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1404 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1405 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1406 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1407 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1408 }
1409 if (i < count) {
1410 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1411 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1412 vsf = _mm_mul_ps(vsf, vsa);
1413 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1414 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1415 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1416 __m128i vsi = _mm_cvttps_epi32(vsf);
1417 vsi = _mm_packus_epi32(vsi, vsi);
1418 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1419 vsi = _mm_packus_epi16(vsi, vsi);
1420 buffer[i] = _mm_cvtsi128_si32(vsi);
1421 }
1422 return buffer;
1423}
1424
1425const QRgba64 *QT_FASTCALL fetchRGBA16FPMToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1426 const QList<QRgb> *, QDitherInfo *)
1427{
1428 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1429 const __m256 vf = _mm256_set1_ps(65535.0f);
1430 const __m256 vh = _mm256_set1_ps(0.5f);
1431 int i = 0;
1432 for (; i + 1 < count; i += 2) {
1433 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1434 vsf = _mm256_mul_ps(vsf, vf);
1435 vsf = _mm256_add_ps(vsf, vh);
1436 __m256i vsi = _mm256_cvttps_epi32(vsf);
1437 vsi = _mm256_packus_epi32(vsi, vsi);
1438 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1439 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1440 }
1441 if (i < count) {
1442 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1443 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1444 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1445 __m128i vsi = _mm_cvttps_epi32(vsf);
1446 vsi = _mm_packus_epi32(vsi, vsi);
1447 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1448 }
1449 return buffer;
1450}
1451
1452const QRgba64 *QT_FASTCALL fetchRGBA16FToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1453 const QList<QRgb> *, QDitherInfo *)
1454{
1455 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1456 const __m256 vf = _mm256_set1_ps(65535.0f);
1457 const __m256 vh = _mm256_set1_ps(0.5f);
1458 int i = 0;
1459 for (; i + 1 < count; i += 2) {
1460 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1461 __m256 vsa = _mm256_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1462 vsf = _mm256_mul_ps(vsf, vsa);
1463 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1464 vsf = _mm256_mul_ps(vsf, vf);
1465 vsf = _mm256_add_ps(vsf, vh);
1466 __m256i vsi = _mm256_cvttps_epi32(vsf);
1467 vsi = _mm256_packus_epi32(vsi, vsi);
1468 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1469 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1470 }
1471 if (i < count) {
1472 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1473 __m128 vsa = _mm_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1474 vsf = _mm_mul_ps(vsf, vsa);
1475 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1476 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1477 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1478 __m128i vsi = _mm_cvttps_epi32(vsf);
1479 vsi = _mm_packus_epi32(vsi, vsi);
1480 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1481 }
1482 return buffer;
1483}
1484
1485void QT_FASTCALL storeRGB16FFromRGB32_avx2(uchar *dest, const uint *src, int index, int count,
1486 const QList<QRgb> *, QDitherInfo *)
1487{
1488 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1489 const __m256 vf = _mm256_set1_ps(1.0f / 255.0f);
1490 int i = 0;
1491 for (; i + 1 < count; i += 2) {
1492 __m256i vsi = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(src + i)));
1493 vsi = _mm256_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1494 __m256 vsf = _mm256_cvtepi32_ps(vsi);
1495 vsf = _mm256_mul_ps(vsf, vf);
1496 _mm_storeu_si128((__m128i *)(d + i), _mm256_cvtps_ph(vsf, 0));
1497 }
1498 if (i < count) {
1499 __m128i vsi = _mm_cvtsi32_si128(src[i]);
1500 vsi = _mm_cvtepu8_epi32(vsi);
1501 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1502 __m128 vsf = _mm_cvtepi32_ps(vsi);
1503 vsf = _mm_mul_ps(vsf, _mm_set1_ps(1.0f / 255.0f));
1504 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1505 }
1506}
1507
1508void QT_FASTCALL storeRGBA16FFromARGB32PM_avx2(uchar *dest, const uint *src, int index, int count,
1509 const QList<QRgb> *, QDitherInfo *)
1510{
1511 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1512 const __m128 vf = _mm_set1_ps(1.0f / 255.0f);
1513 for (int i = 0; i < count; ++i) {
1514 const uint s = src[i];
1515 __m128i vsi = _mm_cvtsi32_si128(s);
1516 vsi = _mm_cvtepu8_epi32(vsi);
1517 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1518 __m128 vsf = _mm_cvtepi32_ps(vsi);
1519 const uint8_t a = (s >> 24);
1520 if (a == 255)
1521 vsf = _mm_mul_ps(vsf, vf);
1522 else if (a == 0)
1523 vsf = _mm_set1_ps(0.0f);
1524 else {
1525 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1526 __m128 vsr = _mm_rcp_ps(vsa);
1527 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1528 vsr = _mm_insert_ps(vsr, _mm_set_ss(1.0f), 0x30);
1529 vsf = _mm_mul_ps(vsf, vsr);
1530 }
1531 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1532 }
1533}
1534
1535#if QT_CONFIG(raster_fp)
1536const QRgbaFloat32 *QT_FASTCALL fetchRGBA16FToRGBA32F_avx2(QRgbaFloat32 *buffer, const uchar *src, int index, int count,
1537 const QList<QRgb> *, QDitherInfo *)
1538{
1539 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1540 int i = 0;
1541 for (; i + 1 < count; i += 2) {
1542 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1543 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1544 vsf = _mm256_mul_ps(vsf, vsa);
1545 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1546 _mm256_storeu_ps((float *)(buffer + i), vsf);
1547 }
1548 if (i < count) {
1549 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1550 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1551 vsf = _mm_mul_ps(vsf, vsa);
1552 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1553 _mm_storeu_ps((float *)(buffer + i), vsf);
1554 }
1555 return buffer;
1556}
1557
1558void QT_FASTCALL storeRGBX16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1559 const QList<QRgb> *, QDitherInfo *)
1560{
1561 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1562 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1563 const __m128 zero = _mm_set_ps(1.0f, 0.0f, 0.0f, 0.0f);
1564 for (int i = 0; i < count; ++i) {
1565 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1566 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1567 const float a = _mm_cvtss_f32(vsa);
1568 if (a == 1.0f)
1569 { }
1570 else if (a == 0.0f)
1571 vsf = zero;
1572 else {
1573 __m128 vsr = _mm_rcp_ps(vsa);
1574 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1575 vsf = _mm_mul_ps(vsf, vsr);
1576 vsf = _mm_insert_ps(vsf, _mm_set_ss(1.0f), 0x30);
1577 }
1578 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1579 }
1580}
1581
1582void QT_FASTCALL storeRGBA16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1583 const QList<QRgb> *, QDitherInfo *)
1584{
1585 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1586 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1587 const __m128 zero = _mm_set1_ps(0.0f);
1588 for (int i = 0; i < count; ++i) {
1589 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1590 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1591 const float a = _mm_cvtss_f32(vsa);
1592 if (a == 1.0f)
1593 { }
1594 else if (a == 0.0f)
1595 vsf = zero;
1596 else {
1597 __m128 vsr = _mm_rcp_ps(vsa);
1598 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1599 vsr = _mm_insert_ps(vsr, _mm_set_ss(1.0f), 0x30);
1600 vsf = _mm_mul_ps(vsf, vsr);
1601 }
1602 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1603 }
1604}
1605#endif
1606
1607QT_END_NAMESPACE
1608
1609#endif