Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qdrawhelper_avx2.cpp
Go to the documentation of this file.
1// Copyright (C) 2018 The Qt Company Ltd.
2// Copyright (C) 2018 Intel Corporation.
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
4
5#include "qdrawhelper_p.h"
6#include "qdrawhelper_x86_p.h"
7#include "qdrawingprimitive_sse2_p.h"
8#include "qpixellayout_p.h"
9#include "qrgba64_p.h"
10
11#if defined(QT_COMPILER_SUPPORTS_AVX2)
12
13QT_BEGIN_NAMESPACE
14
15enum {
16 FixedScale = 1 << 16,
17 HalfPoint = 1 << 15
18};
19
20// Vectorized blend functions:
21
22// See BYTE_MUL_SSE2 for details.
23inline static void Q_DECL_VECTORCALL
24BYTE_MUL_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
25{
26 __m256i pixelVectorAG = _mm256_srli_epi16(pixelVector, 8);
27 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
28
29 pixelVectorAG = _mm256_mullo_epi16(pixelVectorAG, alphaChannel);
30 pixelVectorRB = _mm256_mullo_epi16(pixelVectorRB, alphaChannel);
31
32 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, _mm256_srli_epi16(pixelVectorRB, 8));
33 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, _mm256_srli_epi16(pixelVectorAG, 8));
34 pixelVectorRB = _mm256_add_epi16(pixelVectorRB, half);
35 pixelVectorAG = _mm256_add_epi16(pixelVectorAG, half);
36
37 pixelVectorRB = _mm256_srli_epi16(pixelVectorRB, 8);
38 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
39}
40
41#if QT_CONFIG(raster_64bit)
42inline static void Q_DECL_VECTORCALL
43BYTE_MUL_RGB64_AVX2(__m256i &pixelVector, __m256i alphaChannel, __m256i colorMask, __m256i half)
44{
45 __m256i pixelVectorAG = _mm256_srli_epi32(pixelVector, 16);
46 __m256i pixelVectorRB = _mm256_and_si256(pixelVector, colorMask);
47
48 pixelVectorAG = _mm256_mullo_epi32(pixelVectorAG, alphaChannel);
49 pixelVectorRB = _mm256_mullo_epi32(pixelVectorRB, alphaChannel);
50
51 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, _mm256_srli_epi32(pixelVectorRB, 16));
52 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, _mm256_srli_epi32(pixelVectorAG, 16));
53 pixelVectorRB = _mm256_add_epi32(pixelVectorRB, half);
54 pixelVectorAG = _mm256_add_epi32(pixelVectorAG, half);
55
56 pixelVectorRB = _mm256_srli_epi32(pixelVectorRB, 16);
57 pixelVector = _mm256_blendv_epi8(pixelVectorAG, pixelVectorRB, colorMask);
58}
59#endif
60
61// See INTERPOLATE_PIXEL_255_SSE2 for details.
62inline static void Q_DECL_VECTORCALL
63INTERPOLATE_PIXEL_255_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
64{
65 const __m256i srcVectorAG = _mm256_srli_epi16(srcVector, 8);
66 const __m256i dstVectorAG = _mm256_srli_epi16(dstVector, 8);
67 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
68 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
69 const __m256i srcVectorAGalpha = _mm256_mullo_epi16(srcVectorAG, alphaChannel);
70 const __m256i srcVectorRBalpha = _mm256_mullo_epi16(srcVectorRB, alphaChannel);
71 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi16(dstVectorAG, oneMinusAlphaChannel);
72 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi16(dstVectorRB, oneMinusAlphaChannel);
73 __m256i finalAG = _mm256_add_epi16(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
74 __m256i finalRB = _mm256_add_epi16(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
75 finalAG = _mm256_add_epi16(finalAG, _mm256_srli_epi16(finalAG, 8));
76 finalRB = _mm256_add_epi16(finalRB, _mm256_srli_epi16(finalRB, 8));
77 finalAG = _mm256_add_epi16(finalAG, half);
78 finalRB = _mm256_add_epi16(finalRB, half);
79 finalRB = _mm256_srli_epi16(finalRB, 8);
80
81 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
82}
83
84#if QT_CONFIG(raster_64bit)
85inline static void Q_DECL_VECTORCALL
86INTERPOLATE_PIXEL_RGB64_AVX2(__m256i srcVector, __m256i &dstVector, __m256i alphaChannel, __m256i oneMinusAlphaChannel, __m256i colorMask, __m256i half)
87{
88 const __m256i srcVectorAG = _mm256_srli_epi32(srcVector, 16);
89 const __m256i dstVectorAG = _mm256_srli_epi32(dstVector, 16);
90 const __m256i srcVectorRB = _mm256_and_si256(srcVector, colorMask);
91 const __m256i dstVectorRB = _mm256_and_si256(dstVector, colorMask);
92 const __m256i srcVectorAGalpha = _mm256_mullo_epi32(srcVectorAG, alphaChannel);
93 const __m256i srcVectorRBalpha = _mm256_mullo_epi32(srcVectorRB, alphaChannel);
94 const __m256i dstVectorAGoneMinusAlpha = _mm256_mullo_epi32(dstVectorAG, oneMinusAlphaChannel);
95 const __m256i dstVectorRBoneMinusAlpha = _mm256_mullo_epi32(dstVectorRB, oneMinusAlphaChannel);
96 __m256i finalAG = _mm256_add_epi32(srcVectorAGalpha, dstVectorAGoneMinusAlpha);
97 __m256i finalRB = _mm256_add_epi32(srcVectorRBalpha, dstVectorRBoneMinusAlpha);
98 finalAG = _mm256_add_epi32(finalAG, _mm256_srli_epi32(finalAG, 16));
99 finalRB = _mm256_add_epi32(finalRB, _mm256_srli_epi32(finalRB, 16));
100 finalAG = _mm256_add_epi32(finalAG, half);
101 finalRB = _mm256_add_epi32(finalRB, half);
102 finalRB = _mm256_srli_epi32(finalRB, 16);
103 dstVector = _mm256_blendv_epi8(finalAG, finalRB, colorMask);
104}
105#endif
106
107// See BLEND_SOURCE_OVER_ARGB32_SSE2 for details.
108inline static void Q_DECL_VECTORCALL BLEND_SOURCE_OVER_ARGB32_AVX2(quint32 *dst, const quint32 *src, const int length)
109{
110 const __m256i half = _mm256_set1_epi16(0x80);
111 const __m256i one = _mm256_set1_epi16(0xff);
112 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
113 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
114 const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
115 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
116 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
117
118 const int minusOffsetToAlignDstOn32Bytes = (reinterpret_cast<quintptr>(dst) >> 2) & 0x7;
119
120 int x = 0;
121 // Prologue to handle all pixels until dst is 32-byte aligned in one step.
122 if (minusOffsetToAlignDstOn32Bytes != 0 && x < (length - 7)) {
123 const __m256i prologueMask = _mm256_sub_epi32(_mm256_set1_epi32(minusOffsetToAlignDstOn32Bytes - 1), offsetMask);
124 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
125 const __m256i prologueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, prologueMask);
126 if (!_mm256_testz_si256(srcVector, prologueAlphaMask)) {
127 if (_mm256_testc_si256(srcVector, prologueAlphaMask)) {
128 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, srcVector);
129 } else {
130 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
131 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
132 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask);
133 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
134 dstVector = _mm256_add_epi8(dstVector, srcVector);
135 _mm256_maskstore_epi32((int *)&dst[x - minusOffsetToAlignDstOn32Bytes], prologueMask, dstVector);
136 }
137 }
138 x += (8 - minusOffsetToAlignDstOn32Bytes);
139 }
140
141 for (; x < (length - 7); x += 8) {
142 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
143 if (!_mm256_testz_si256(srcVector, alphaMask)) {
144 if (_mm256_testc_si256(srcVector, alphaMask)) {
145 _mm256_store_si256((__m256i *)&dst[x], srcVector);
146 } else {
147 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
148 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
149 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
150 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
151 dstVector = _mm256_add_epi8(dstVector, srcVector);
152 _mm256_store_si256((__m256i *)&dst[x], dstVector);
153 }
154 }
155 }
156
157 // Epilogue to handle all remaining pixels in one step.
158 if (x < length) {
159 const __m256i epilogueMask = _mm256_add_epi32(offsetMask, _mm256_set1_epi32(x - length));
160 const __m256i srcVector = _mm256_maskload_epi32((const int *)&src[x], epilogueMask);
161 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
162 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
163 if (_mm256_testc_si256(srcVector, epilogueAlphaMask)) {
164 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, srcVector);
165 } else {
166 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
167 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
168 __m256i dstVector = _mm256_maskload_epi32((int *)&dst[x], epilogueMask);
169 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
170 dstVector = _mm256_add_epi8(dstVector, srcVector);
171 _mm256_maskstore_epi32((int *)&dst[x], epilogueMask, dstVector);
172 }
173 }
174 }
175}
176
177
178// See BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_SSE2 for details.
179inline static void Q_DECL_VECTORCALL
180BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(quint32 *dst, const quint32 *src, const int length, const int const_alpha)
181{
182 int x = 0;
183
184 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
185 blend_pixel(dst[x], src[x], const_alpha);
186
187 const __m256i half = _mm256_set1_epi16(0x80);
188 const __m256i one = _mm256_set1_epi16(0xff);
189 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
190 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
191 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3,
192 char(0xff),15,char(0xff),15,char(0xff),11,char(0xff),11,char(0xff),7,char(0xff),7,char(0xff),3,char(0xff),3);
193 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
194 for (; x < (length - 7); x += 8) {
195 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
196 if (!_mm256_testz_si256(srcVector, alphaMask)) {
197 BYTE_MUL_AVX2(srcVector, constAlphaVector, colorMask, half);
198
199 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
200 alphaChannel = _mm256_sub_epi16(one, alphaChannel);
201 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
202 BYTE_MUL_AVX2(dstVector, alphaChannel, colorMask, half);
203 dstVector = _mm256_add_epi8(dstVector, srcVector);
204 _mm256_store_si256((__m256i *)&dst[x], dstVector);
205 }
206 }
207 SIMD_EPILOGUE(x, length, 7)
208 blend_pixel(dst[x], src[x], const_alpha);
209}
210
211void qt_blend_argb32_on_argb32_avx2(uchar *destPixels, int dbpl,
212 const uchar *srcPixels, int sbpl,
213 int w, int h,
214 int const_alpha)
215{
216 if (const_alpha == 256) {
217 for (int y = 0; y < h; ++y) {
218 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
219 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
220 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, w);
221 destPixels += dbpl;
222 srcPixels += sbpl;
223 }
224 } else if (const_alpha != 0) {
225 const_alpha = (const_alpha * 255) >> 8;
226 for (int y = 0; y < h; ++y) {
227 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
228 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
229 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, w, const_alpha);
230 destPixels += dbpl;
231 srcPixels += sbpl;
232 }
233 }
234}
235
236void qt_blend_rgb32_on_rgb32_avx2(uchar *destPixels, int dbpl,
237 const uchar *srcPixels, int sbpl,
238 int w, int h,
239 int const_alpha)
240{
241 if (const_alpha == 256) {
242 for (int y = 0; y < h; ++y) {
243 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
244 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
245 ::memcpy(dst, src, w * sizeof(uint));
246 srcPixels += sbpl;
247 destPixels += dbpl;
248 }
249 return;
250 }
251 if (const_alpha == 0)
252 return;
253
254 const __m256i half = _mm256_set1_epi16(0x80);
255 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
256
257 const_alpha = (const_alpha * 255) >> 8;
258 int one_minus_const_alpha = 255 - const_alpha;
259 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
260 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(one_minus_const_alpha);
261 for (int y = 0; y < h; ++y) {
262 const quint32 *src = reinterpret_cast<const quint32 *>(srcPixels);
263 quint32 *dst = reinterpret_cast<quint32 *>(destPixels);
264 int x = 0;
265
266 // First, align dest to 32 bytes:
267 ALIGNMENT_PROLOGUE_32BYTES(dst, x, w)
268 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
269
270 // 2) interpolate pixels with AVX2
271 for (; x < (w - 7); x += 8) {
272 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
273 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
274 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
275 _mm256_store_si256((__m256i *)&dst[x], dstVector);
276 }
277
278 // 3) Epilogue
279 SIMD_EPILOGUE(x, w, 7)
280 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
281
282 srcPixels += sbpl;
283 destPixels += dbpl;
284 }
285}
286
287Q_NEVER_INLINE static
288void Q_DECL_VECTORCALL qt_memfillXX_avx2(uchar *dest, __m256i value256, qsizetype bytes)
289{
290 __m128i value128 = _mm256_castsi256_si128(value256);
291
292 // main body
293 __m256i *dst256 = reinterpret_cast<__m256i *>(dest);
294 uchar *end = dest + bytes;
295 while (reinterpret_cast<uchar *>(dst256 + 4) <= end) {
296 _mm256_storeu_si256(dst256 + 0, value256);
297 _mm256_storeu_si256(dst256 + 1, value256);
298 _mm256_storeu_si256(dst256 + 2, value256);
299 _mm256_storeu_si256(dst256 + 3, value256);
300 dst256 += 4;
301 }
302
303 // first epilogue: fewer than 128 bytes / 32 entries
304 bytes = end - reinterpret_cast<uchar *>(dst256);
305 switch (bytes / sizeof(value256)) {
306 case 3: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
307 case 2: _mm256_storeu_si256(dst256++, value256); Q_FALLTHROUGH();
308 case 1: _mm256_storeu_si256(dst256++, value256);
309 }
310
311 // second epilogue: fewer than 32 bytes
312 __m128i *dst128 = reinterpret_cast<__m128i *>(dst256);
313 if (bytes & sizeof(value128))
314 _mm_storeu_si128(dst128++, value128);
315
316 // third epilogue: fewer than 16 bytes
317 if (bytes & 8)
318 _mm_storel_epi64(reinterpret_cast<__m128i *>(end - 8), value128);
319}
320
321void qt_memfill64_avx2(quint64 *dest, quint64 value, qsizetype count)
322{
323#if defined(Q_CC_GNU) && !defined(Q_CC_CLANG)
324 // work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80820
325 __m128i value64 = _mm_set_epi64x(0, value); // _mm_cvtsi64_si128(value);
326# ifdef Q_PROCESSOR_X86_64
327 asm ("" : "+x" (value64));
328# endif
329 __m256i value256 = _mm256_broadcastq_epi64(value64);
330#else
331 __m256i value256 = _mm256_set1_epi64x(value);
332#endif
333
334 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), value256, count * sizeof(quint64));
335}
336
337void qt_memfill32_avx2(quint32 *dest, quint32 value, qsizetype count)
338{
339 if (count % 2) {
340 // odd number of pixels, round to even
341 *dest++ = value;
342 --count;
343 }
344 qt_memfillXX_avx2(reinterpret_cast<uchar *>(dest), _mm256_set1_epi32(value), count * sizeof(quint32));
345}
346
347void QT_FASTCALL comp_func_SourceOver_avx2(uint *destPixels, const uint *srcPixels, int length, uint const_alpha)
348{
349 Q_ASSERT(const_alpha < 256);
350
351 const quint32 *src = (const quint32 *) srcPixels;
352 quint32 *dst = (quint32 *) destPixels;
353
354 if (const_alpha == 255)
355 BLEND_SOURCE_OVER_ARGB32_AVX2(dst, src, length);
356 else
357 BLEND_SOURCE_OVER_ARGB32_WITH_CONST_ALPHA_AVX2(dst, src, length, const_alpha);
358}
359
360#if QT_CONFIG(raster_64bit)
361void QT_FASTCALL comp_func_SourceOver_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
362{
363 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
364 const __m256i half = _mm256_set1_epi32(0x8000);
365 const __m256i one = _mm256_set1_epi32(0xffff);
366 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
367 __m256i alphaMask = _mm256_set1_epi32(0xff000000);
368 alphaMask = _mm256_unpacklo_epi8(alphaMask, alphaMask);
369 const __m256i alphaShuffleMask = _mm256_set_epi8(char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6,
370 char(0xff),char(0xff),15,14,char(0xff),char(0xff),15,14,char(0xff),char(0xff),7,6,char(0xff),char(0xff),7,6);
371
372 if (const_alpha == 255) {
373 int x = 0;
374 for (; x < length && (quintptr(dst + x) & 31); ++x)
375 blend_pixel(dst[x], src[x]);
376 for (; x < length - 3; x += 4) {
377 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
378 if (!_mm256_testz_si256(srcVector, alphaMask)) {
379 // Not all transparent
380 if (_mm256_testc_si256(srcVector, alphaMask)) {
381 // All opaque
382 _mm256_store_si256((__m256i *)&dst[x], srcVector);
383 } else {
384 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
385 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
386 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
387 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
388 dstVector = _mm256_add_epi16(dstVector, srcVector);
389 _mm256_store_si256((__m256i *)&dst[x], dstVector);
390 }
391 }
392 }
393 SIMD_EPILOGUE(x, length, 3)
394 blend_pixel(dst[x], src[x]);
395 } else {
396 const __m256i constAlphaVector = _mm256_set1_epi32(const_alpha | (const_alpha << 8));
397 int x = 0;
398 for (; x < length && (quintptr(dst + x) & 31); ++x)
399 blend_pixel(dst[x], src[x], const_alpha);
400 for (; x < length - 3; x += 4) {
401 __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
402 if (!_mm256_testz_si256(srcVector, alphaMask)) {
403 // Not all transparent
404 BYTE_MUL_RGB64_AVX2(srcVector, constAlphaVector, colorMask, half);
405
406 __m256i alphaChannel = _mm256_shuffle_epi8(srcVector, alphaShuffleMask);
407 alphaChannel = _mm256_sub_epi32(one, alphaChannel);
408 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
409 BYTE_MUL_RGB64_AVX2(dstVector, alphaChannel, colorMask, half);
410 dstVector = _mm256_add_epi16(dstVector, srcVector);
411 _mm256_store_si256((__m256i *)&dst[x], dstVector);
412 }
413 }
414 SIMD_EPILOGUE(x, length, 3)
415 blend_pixel(dst[x], src[x], const_alpha);
416 }
417}
418#endif
419
420#if QT_CONFIG(raster_fp)
421void QT_FASTCALL comp_func_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
422{
423 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
424
425 const float a = const_alpha / 255.0f;
426 const __m128 one = _mm_set1_ps(1.0f);
427 const __m128 constAlphaVector = _mm_set1_ps(a);
428 const __m256 one256 = _mm256_set1_ps(1.0f);
429 const __m256 constAlphaVector256 = _mm256_set1_ps(a);
430 int x = 0;
431 for (; x < length - 1; x += 2) {
432 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
433 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
434 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
435 __m256 alphaChannel = _mm256_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
436 alphaChannel = _mm256_sub_ps(one256, alphaChannel);
437 dstVector = _mm256_mul_ps(dstVector, alphaChannel);
438 dstVector = _mm256_add_ps(dstVector, srcVector);
439 _mm256_storeu_ps((float *)(dst + x), dstVector);
440 }
441 if (x < length) {
442 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
443 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
444 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
445 __m128 alphaChannel = _mm_permute_ps(srcVector, _MM_SHUFFLE(3, 3, 3, 3));
446 alphaChannel = _mm_sub_ps(one, alphaChannel);
447 dstVector = _mm_mul_ps(dstVector, alphaChannel);
448 dstVector = _mm_add_ps(dstVector, srcVector);
449 _mm_storeu_ps((float *)(dst + x), dstVector);
450 }
451}
452#endif
453
454void QT_FASTCALL comp_func_Source_avx2(uint *dst, const uint *src, int length, uint const_alpha)
455{
456 if (const_alpha == 255) {
457 ::memcpy(dst, src, length * sizeof(uint));
458 } else {
459 const int ialpha = 255 - const_alpha;
460
461 int x = 0;
462
463 // 1) prologue, align on 32 bytes
464 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
465 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
466
467 // 2) interpolate pixels with AVX2
468 const __m256i half = _mm256_set1_epi16(0x80);
469 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
470 const __m256i constAlphaVector = _mm256_set1_epi16(const_alpha);
471 const __m256i oneMinusConstAlpha = _mm256_set1_epi16(ialpha);
472 for (; x < length - 7; x += 8) {
473 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
474 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
475 INTERPOLATE_PIXEL_255_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
476 _mm256_store_si256((__m256i *)&dst[x], dstVector);
477 }
478
479 // 3) Epilogue
480 SIMD_EPILOGUE(x, length, 7)
481 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], ialpha);
482 }
483}
484
485#if QT_CONFIG(raster_64bit)
486void QT_FASTCALL comp_func_Source_rgb64_avx2(QRgba64 *dst, const QRgba64 *src, int length, uint const_alpha)
487{
488 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
489 if (const_alpha == 255) {
490 ::memcpy(dst, src, length * sizeof(QRgba64));
491 } else {
492 const uint ca = const_alpha | (const_alpha << 8); // adjust to [0-65535]
493 const uint cia = 65535 - ca;
494
495 int x = 0;
496
497 // 1) prologue, align on 32 bytes
498 for (; x < length && (quintptr(dst + x) & 31); ++x)
499 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
500
501 // 2) interpolate pixels with AVX2
502 const __m256i half = _mm256_set1_epi32(0x8000);
503 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
504 const __m256i constAlphaVector = _mm256_set1_epi32(ca);
505 const __m256i oneMinusConstAlpha = _mm256_set1_epi32(cia);
506 for (; x < length - 3; x += 4) {
507 const __m256i srcVector = _mm256_lddqu_si256((const __m256i *)&src[x]);
508 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
509 INTERPOLATE_PIXEL_RGB64_AVX2(srcVector, dstVector, constAlphaVector, oneMinusConstAlpha, colorMask, half);
510 _mm256_store_si256((__m256i *)&dst[x], dstVector);
511 }
512
513 // 3) Epilogue
514 SIMD_EPILOGUE(x, length, 3)
515 dst[x] = interpolate65535(src[x], ca, dst[x], cia);
516 }
517}
518#endif
519
520#if QT_CONFIG(raster_fp)
521void QT_FASTCALL comp_func_Source_rgbafp_avx2(QRgbaFloat32 *dst, const QRgbaFloat32 *src, int length, uint const_alpha)
522{
523 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
524 if (const_alpha == 255) {
525 ::memcpy(dst, src, length * sizeof(QRgbaFloat32));
526 } else {
527 const float ca = const_alpha / 255.f;
528 const float cia = 1.0f - ca;
529
530 const __m128 constAlphaVector = _mm_set1_ps(ca);
531 const __m128 oneMinusConstAlpha = _mm_set1_ps(cia);
532 const __m256 constAlphaVector256 = _mm256_set1_ps(ca);
533 const __m256 oneMinusConstAlpha256 = _mm256_set1_ps(cia);
534 int x = 0;
535 for (; x < length - 1; x += 2) {
536 __m256 srcVector = _mm256_loadu_ps((const float *)&src[x]);
537 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
538 srcVector = _mm256_mul_ps(srcVector, constAlphaVector256);
539 dstVector = _mm256_mul_ps(dstVector, oneMinusConstAlpha256);
540 dstVector = _mm256_add_ps(dstVector, srcVector);
541 _mm256_storeu_ps((float *)&dst[x], dstVector);
542 }
543 if (x < length) {
544 __m128 srcVector = _mm_loadu_ps((const float *)&src[x]);
545 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
546 srcVector = _mm_mul_ps(srcVector, constAlphaVector);
547 dstVector = _mm_mul_ps(dstVector, oneMinusConstAlpha);
548 dstVector = _mm_add_ps(dstVector, srcVector);
549 _mm_storeu_ps((float *)&dst[x], dstVector);
550 }
551 }
552}
553#endif
554
555void QT_FASTCALL comp_func_solid_SourceOver_avx2(uint *destPixels, int length, uint color, uint const_alpha)
556{
557 if ((const_alpha & qAlpha(color)) == 255) {
558 qt_memfill32(destPixels, color, length);
559 } else {
560 if (const_alpha != 255)
561 color = BYTE_MUL(color, const_alpha);
562
563 const quint32 minusAlphaOfColor = qAlpha(~color);
564 int x = 0;
565
566 quint32 *dst = (quint32 *) destPixels;
567 const __m256i colorVector = _mm256_set1_epi32(color);
568 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
569 const __m256i half = _mm256_set1_epi16(0x80);
570 const __m256i minusAlphaOfColorVector = _mm256_set1_epi16(minusAlphaOfColor);
571
572 ALIGNMENT_PROLOGUE_32BYTES(dst, x, length)
573 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
574
575 for (; x < length - 7; x += 8) {
576 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
577 BYTE_MUL_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
578 dstVector = _mm256_add_epi8(colorVector, dstVector);
579 _mm256_store_si256((__m256i *)&dst[x], dstVector);
580 }
581 SIMD_EPILOGUE(x, length, 7)
582 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
583 }
584}
585
586#if QT_CONFIG(raster_64bit)
587void QT_FASTCALL comp_func_solid_SourceOver_rgb64_avx2(QRgba64 *destPixels, int length, QRgba64 color, uint const_alpha)
588{
589 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
590 if (const_alpha == 255 && color.isOpaque()) {
591 qt_memfill64((quint64*)destPixels, color, length);
592 } else {
593 if (const_alpha != 255)
594 color = multiplyAlpha255(color, const_alpha);
595
596 const uint minusAlphaOfColor = 65535 - color.alpha();
597 int x = 0;
598 quint64 *dst = (quint64 *) destPixels;
599 const __m256i colorVector = _mm256_set1_epi64x(color);
600 const __m256i colorMask = _mm256_set1_epi32(0x0000ffff);
601 const __m256i half = _mm256_set1_epi32(0x8000);
602 const __m256i minusAlphaOfColorVector = _mm256_set1_epi32(minusAlphaOfColor);
603
604 for (; x < length && (quintptr(dst + x) & 31); ++x)
605 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
606
607 for (; x < length - 3; x += 4) {
608 __m256i dstVector = _mm256_load_si256((__m256i *)&dst[x]);
609 BYTE_MUL_RGB64_AVX2(dstVector, minusAlphaOfColorVector, colorMask, half);
610 dstVector = _mm256_add_epi16(colorVector, dstVector);
611 _mm256_store_si256((__m256i *)&dst[x], dstVector);
612 }
613 SIMD_EPILOGUE(x, length, 3)
614 destPixels[x] = color + multiplyAlpha65535(destPixels[x], minusAlphaOfColor);
615 }
616}
617#endif
618
619#if QT_CONFIG(raster_fp)
620void QT_FASTCALL comp_func_solid_Source_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
621{
622 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
623 if (const_alpha == 255) {
624 for (int i = 0; i < length; ++i)
625 dst[i] = color;
626 } else {
627 const float a = const_alpha / 255.0f;
628 const __m128 alphaVector = _mm_set1_ps(a);
629 const __m128 minusAlphaVector = _mm_set1_ps(1.0f - a);
630 __m128 colorVector = _mm_loadu_ps((const float *)&color);
631 colorVector = _mm_mul_ps(colorVector, alphaVector);
632 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
633 const __m256 minusAlphaVector256 = _mm256_set1_ps(1.0f - a);
634 int x = 0;
635 for (; x < length - 1; x += 2) {
636 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
637 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
638 dstVector = _mm256_add_ps(dstVector, colorVector256);
639 _mm256_storeu_ps((float *)&dst[x], dstVector);
640 }
641 if (x < length) {
642 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
643 dstVector = _mm_mul_ps(dstVector, minusAlphaVector);
644 dstVector = _mm_add_ps(dstVector, colorVector);
645 _mm_storeu_ps((float *)&dst[x], dstVector);
646 }
647 }
648}
649
650void QT_FASTCALL comp_func_solid_SourceOver_rgbafp_avx2(QRgbaFloat32 *dst, int length, QRgbaFloat32 color, uint const_alpha)
651{
652 Q_ASSERT(const_alpha < 256); // const_alpha is in [0-255]
653 if (const_alpha == 255 && color.a >= 1.0f) {
654 for (int i = 0; i < length; ++i)
655 dst[i] = color;
656 } else {
657 __m128 colorVector = _mm_loadu_ps((const float *)&color);
658 if (const_alpha != 255)
659 colorVector = _mm_mul_ps(colorVector, _mm_set1_ps(const_alpha / 255.f));
660 __m128 minusAlphaOfColorVector =
661 _mm_sub_ps(_mm_set1_ps(1.0f), _mm_permute_ps(colorVector, _MM_SHUFFLE(3, 3, 3, 3)));
662 const __m256 colorVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(colorVector), colorVector, 1);
663 const __m256 minusAlphaVector256 = _mm256_insertf128_ps(_mm256_castps128_ps256(minusAlphaOfColorVector),
664 minusAlphaOfColorVector, 1);
665 int x = 0;
666 for (; x < length - 1; x += 2) {
667 __m256 dstVector = _mm256_loadu_ps((const float *)&dst[x]);
668 dstVector = _mm256_mul_ps(dstVector, minusAlphaVector256);
669 dstVector = _mm256_add_ps(dstVector, colorVector256);
670 _mm256_storeu_ps((float *)&dst[x], dstVector);
671 }
672 if (x < length) {
673 __m128 dstVector = _mm_loadu_ps((const float *)&dst[x]);
674 dstVector = _mm_mul_ps(dstVector, minusAlphaOfColorVector);
675 dstVector = _mm_add_ps(dstVector, colorVector);
676 _mm_storeu_ps((float *)&dst[x], dstVector);
677 }
678 }
679}
680#endif
681
682#define interpolate_4_pixels_16_avx2(tlr1, tlr2, blr1, blr2, distx, disty, colorMask, v_256, b) \
683{
684 /* Correct for later unpack */
685 const __m256i vdistx = _mm256_permute4x64_epi64(distx, _MM_SHUFFLE(3, 1, 2, 0));
686 const __m256i vdisty = _mm256_permute4x64_epi64(disty, _MM_SHUFFLE(3, 1, 2, 0));
687
688 __m256i dxdy = _mm256_mullo_epi16 (vdistx, vdisty);
689 const __m256i distx_ = _mm256_slli_epi16(vdistx, 4);
690 const __m256i disty_ = _mm256_slli_epi16(vdisty, 4);
691 __m256i idxidy = _mm256_add_epi16(dxdy, _mm256_sub_epi16(v_256, _mm256_add_epi16(distx_, disty_)));
692 __m256i dxidy = _mm256_sub_epi16(distx_, dxdy);
693 __m256i idxdy = _mm256_sub_epi16(disty_, dxdy);
694
695 __m256i tlr1AG = _mm256_srli_epi16(tlr1, 8);
696 __m256i tlr1RB = _mm256_and_si256(tlr1, colorMask);
697 __m256i tlr2AG = _mm256_srli_epi16(tlr2, 8);
698 __m256i tlr2RB = _mm256_and_si256(tlr2, colorMask);
699 __m256i blr1AG = _mm256_srli_epi16(blr1, 8);
700 __m256i blr1RB = _mm256_and_si256(blr1, colorMask);
701 __m256i blr2AG = _mm256_srli_epi16(blr2, 8);
702 __m256i blr2RB = _mm256_and_si256(blr2, colorMask);
703
704 __m256i odxidy1 = _mm256_unpacklo_epi32(idxidy, dxidy);
705 __m256i odxidy2 = _mm256_unpackhi_epi32(idxidy, dxidy);
706 tlr1AG = _mm256_mullo_epi16(tlr1AG, odxidy1);
707 tlr1RB = _mm256_mullo_epi16(tlr1RB, odxidy1);
708 tlr2AG = _mm256_mullo_epi16(tlr2AG, odxidy2);
709 tlr2RB = _mm256_mullo_epi16(tlr2RB, odxidy2);
710 __m256i odxdy1 = _mm256_unpacklo_epi32(idxdy, dxdy);
711 __m256i odxdy2 = _mm256_unpackhi_epi32(idxdy, dxdy);
712 blr1AG = _mm256_mullo_epi16(blr1AG, odxdy1);
713 blr1RB = _mm256_mullo_epi16(blr1RB, odxdy1);
714 blr2AG = _mm256_mullo_epi16(blr2AG, odxdy2);
715 blr2RB = _mm256_mullo_epi16(blr2RB, odxdy2);
716
717 /* Add the values, and shift to only keep 8 significant bits per colors */
718 __m256i topAG = _mm256_hadd_epi32(tlr1AG, tlr2AG);
719 __m256i topRB = _mm256_hadd_epi32(tlr1RB, tlr2RB);
720 __m256i botAG = _mm256_hadd_epi32(blr1AG, blr2AG);
721 __m256i botRB = _mm256_hadd_epi32(blr1RB, blr2RB);
722 __m256i rAG = _mm256_add_epi16(topAG, botAG);
723 __m256i rRB = _mm256_add_epi16(topRB, botRB);
724 rRB = _mm256_srli_epi16(rRB, 8);
725 /* Correct for hadd */
726 rAG = _mm256_permute4x64_epi64(rAG, _MM_SHUFFLE(3, 1, 2, 0));
727 rRB = _mm256_permute4x64_epi64(rRB, _MM_SHUFFLE(3, 1, 2, 0));
728 _mm256_storeu_si256((__m256i*)(b), _mm256_blendv_epi8(rAG, rRB, colorMask)); \
729}
730
731inline void fetchTransformedBilinear_pixelBounds(int, int l1, int l2, int &v1, int &v2)
732{
733 if (v1 < l1)
734 v2 = v1 = l1;
735 else if (v1 >= l2)
736 v2 = v1 = l2;
737 else
738 v2 = v1 + 1;
739 Q_ASSERT(v1 >= l1 && v1 <= l2);
740 Q_ASSERT(v2 >= l1 && v2 <= l2);
741}
742
743void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx);
744
745void QT_FASTCALL fetchTransformedBilinearARGB32PM_simple_scale_helper_avx2(uint *b, uint *end, const QTextureData &image,
746 int &fx, int &fy, int fdx, int /*fdy*/)
747{
748 int y1 = (fy >> 16);
749 int y2;
750 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
751 const uint *s1 = (const uint *)image.scanLine(y1);
752 const uint *s2 = (const uint *)image.scanLine(y2);
753
754 const int disty = (fy & 0x0000ffff) >> 8;
755 const int idisty = 256 - disty;
756 const int length = end - b;
757
758 // The intermediate buffer is generated in the positive direction
759 const int adjust = (fdx < 0) ? fdx * length : 0;
760 const int offset = (fx + adjust) >> 16;
761 int x = offset;
762
763 Q_DECL_UNINITIALIZED IntermediateBuffer intermediate;
764 // count is the size used in the intermediate_buffer.
765 int count = (qint64(length) * qAbs(fdx) + FixedScale - 1) / FixedScale + 2;
766 // length is supposed to be <= BufferSize either because data->m11 < 1 or
767 // data->m11 < 2, and any larger buffers split
768 Q_ASSERT(count <= BufferSize + 2);
769 int f = 0;
770 int lim = qMin(count, image.x2 - x);
771 if (x < image.x1) {
772 Q_ASSERT(x < image.x2);
773 uint t = s1[image.x1];
774 uint b = s2[image.x1];
775 quint32 rb = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
776 quint32 ag = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
777 do {
778 intermediate.buffer_rb[f] = rb;
779 intermediate.buffer_ag[f] = ag;
780 f++;
781 x++;
782 } while (x < image.x1 && f < lim);
783 }
784
785 const __m256i disty_ = _mm256_set1_epi16(disty);
786 const __m256i idisty_ = _mm256_set1_epi16(idisty);
787 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
788
789 lim -= 7;
790 for (; f < lim; x += 8, f += 8) {
791 // Load 8 pixels from s1, and split the alpha-green and red-blue component
792 __m256i top = _mm256_loadu_si256((const __m256i*)((const uint *)(s1)+x));
793 __m256i topAG = _mm256_srli_epi16(top, 8);
794 __m256i topRB = _mm256_and_si256(top, colorMask);
795 // Multiplies each color component by idisty
796 topAG = _mm256_mullo_epi16 (topAG, idisty_);
797 topRB = _mm256_mullo_epi16 (topRB, idisty_);
798
799 // Same for the s2 vector
800 __m256i bottom = _mm256_loadu_si256((const __m256i*)((const uint *)(s2)+x));
801 __m256i bottomAG = _mm256_srli_epi16(bottom, 8);
802 __m256i bottomRB = _mm256_and_si256(bottom, colorMask);
803 bottomAG = _mm256_mullo_epi16 (bottomAG, disty_);
804 bottomRB = _mm256_mullo_epi16 (bottomRB, disty_);
805
806 // Add the values, and shift to only keep 8 significant bits per colors
807 __m256i rAG =_mm256_add_epi16(topAG, bottomAG);
808 rAG = _mm256_srli_epi16(rAG, 8);
809 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_ag[f]), rAG);
810 __m256i rRB =_mm256_add_epi16(topRB, bottomRB);
811 rRB = _mm256_srli_epi16(rRB, 8);
812 _mm256_storeu_si256((__m256i*)(&intermediate.buffer_rb[f]), rRB);
813 }
814
815 for (; f < count; f++) { // Same as above but without simd
816 x = qMin(x, image.x2 - 1);
817
818 uint t = s1[x];
819 uint b = s2[x];
820
821 intermediate.buffer_rb[f] = (((t & 0xff00ff) * idisty + (b & 0xff00ff) * disty) >> 8) & 0xff00ff;
822 intermediate.buffer_ag[f] = ((((t>>8) & 0xff00ff) * idisty + ((b>>8) & 0xff00ff) * disty) >> 8) & 0xff00ff;
823 x++;
824 }
825
826 // Now interpolate the values from the intermediate_buffer to get the final result.
827 intermediate_adder_avx2(b, end, intermediate, offset, fx, fdx);
828}
829
830void QT_FASTCALL intermediate_adder_avx2(uint *b, uint *end, const IntermediateBuffer &intermediate, int offset, int &fx, int fdx)
831{
832 fx -= offset * FixedScale;
833
834 const __m128i v_fdx = _mm_set1_epi32(fdx * 4);
835 const __m128i v_blend = _mm_set1_epi32(0x00800080);
836 const __m128i vdx_shuffle = _mm_set_epi8(char(0x80), 13, char(0x80), 13, char(0x80), 9, char(0x80), 9,
837 char(0x80), 5, char(0x80), 5, char(0x80), 1, char(0x80), 1);
838 __m128i v_fx = _mm_setr_epi32(fx, fx + fdx, fx + fdx + fdx, fx + fdx + fdx + fdx);
839
840 while (b < end - 3) {
841 const __m128i offset = _mm_srli_epi32(v_fx, 16);
842 __m256i vrb = _mm256_i32gather_epi64((const long long *)intermediate.buffer_rb, offset, 4);
843 __m256i vag = _mm256_i32gather_epi64((const long long *)intermediate.buffer_ag, offset, 4);
844
845 __m128i vdx = _mm_shuffle_epi8(v_fx, vdx_shuffle);
846 __m128i vidx = _mm_sub_epi16(_mm_set1_epi16(256), vdx);
847 __m256i vmulx = _mm256_castsi128_si256(_mm_unpacklo_epi32(vidx, vdx));
848 vmulx = _mm256_inserti128_si256(vmulx, _mm_unpackhi_epi32(vidx, vdx), 1);
849
850 vrb = _mm256_mullo_epi16(vrb, vmulx);
851 vag = _mm256_mullo_epi16(vag, vmulx);
852
853 __m256i vrbag = _mm256_hadd_epi32(vrb, vag);
854 vrbag = _mm256_permute4x64_epi64(vrbag, _MM_SHUFFLE(3, 1, 2, 0));
855
856 __m128i rb = _mm256_castsi256_si128(vrbag);
857 __m128i ag = _mm256_extracti128_si256(vrbag, 1);
858 rb = _mm_srli_epi16(rb, 8);
859
860 _mm_storeu_si128((__m128i*)b, _mm_blendv_epi8(ag, rb, v_blend));
861
862 b += 4;
863 v_fx = _mm_add_epi32(v_fx, v_fdx);
864 }
865 fx = _mm_cvtsi128_si32(v_fx);
866 while (b < end) {
867 const int x = (fx >> 16);
868
869 const uint distx = (fx & 0x0000ffff) >> 8;
870 const uint idistx = 256 - distx;
871 const uint rb = (intermediate.buffer_rb[x] * idistx + intermediate.buffer_rb[x + 1] * distx) & 0xff00ff00;
872 const uint ag = (intermediate.buffer_ag[x] * idistx + intermediate.buffer_ag[x + 1] * distx) & 0xff00ff00;
873 *b = (rb >> 8) | ag;
874 b++;
875 fx += fdx;
876 }
877 fx += offset * FixedScale;
878}
879
880void QT_FASTCALL fetchTransformedBilinearARGB32PM_downscale_helper_avx2(uint *b, uint *end, const QTextureData &image,
881 int &fx, int &fy, int fdx, int /*fdy*/)
882{
883 int y1 = (fy >> 16);
884 int y2;
885 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
886 const uint *s1 = (const uint *)image.scanLine(y1);
887 const uint *s2 = (const uint *)image.scanLine(y2);
888 const int disty8 = (fy & 0x0000ffff) >> 8;
889 const int disty4 = (disty8 + 0x08) >> 4;
890
891 const qint64 min_fx = qint64(image.x1) * FixedScale;
892 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
893 while (b < end) {
894 int x1 = (fx >> 16);
895 int x2;
896 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
897 if (x1 != x2)
898 break;
899 uint top = s1[x1];
900 uint bot = s2[x1];
901 *b = INTERPOLATE_PIXEL_256(top, 256 - disty8, bot, disty8);
902 fx += fdx;
903 ++b;
904 }
905 uint *boundedEnd = end;
906 if (fdx > 0)
907 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
908 else if (fdx < 0)
909 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
910
911 // A fast middle part without boundary checks
912 const __m256i vdistShuffle =
913 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
914 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
915 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
916 const __m256i v_256 = _mm256_set1_epi16(256);
917 const __m256i v_disty = _mm256_set1_epi16(disty4);
918 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
919 const __m256i v_fx_r = _mm256_set1_epi32(0x08);
920 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
921 __m256i v_fx = _mm256_set1_epi32(fx);
922 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
923
924 while (b < boundedEnd - 7) {
925 const __m256i offset = _mm256_srli_epi32(v_fx, 16);
926 const __m128i offsetLo = _mm256_castsi256_si128(offset);
927 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
928 const __m256i toplo = _mm256_i32gather_epi64((const long long *)s1, offsetLo, 4);
929 const __m256i tophi = _mm256_i32gather_epi64((const long long *)s1, offsetHi, 4);
930 const __m256i botlo = _mm256_i32gather_epi64((const long long *)s2, offsetLo, 4);
931 const __m256i bothi = _mm256_i32gather_epi64((const long long *)s2, offsetHi, 4);
932
933 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
934 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fx_r), 4);
935 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
936
937 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
938 b += 8;
939 v_fx = _mm256_add_epi32(v_fx, v_fdx);
940 }
941 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
942
943 while (b < boundedEnd) {
944 int x = (fx >> 16);
945 int distx8 = (fx & 0x0000ffff) >> 8;
946 *b = interpolate_4_pixels(s1 + x, s2 + x, distx8, disty8);
947 fx += fdx;
948 ++b;
949 }
950
951 while (b < end) {
952 int x1 = (fx >> 16);
953 int x2;
954 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
955 uint tl = s1[x1];
956 uint tr = s1[x2];
957 uint bl = s2[x1];
958 uint br = s2[x2];
959 int distx8 = (fx & 0x0000ffff) >> 8;
960 *b = interpolate_4_pixels(tl, tr, bl, br, distx8, disty8);
961 fx += fdx;
962 ++b;
963 }
964}
965
966void QT_FASTCALL fetchTransformedBilinearARGB32PM_fast_rotate_helper_avx2(uint *b, uint *end, const QTextureData &image,
967 int &fx, int &fy, int fdx, int fdy)
968{
969 const qint64 min_fx = qint64(image.x1) * FixedScale;
970 const qint64 max_fx = qint64(image.x2 - 1) * FixedScale;
971 const qint64 min_fy = qint64(image.y1) * FixedScale;
972 const qint64 max_fy = qint64(image.y2 - 1) * FixedScale;
973 // first handle the possibly bounded part in the beginning
974 while (b < end) {
975 int x1 = (fx >> 16);
976 int x2;
977 int y1 = (fy >> 16);
978 int y2;
979 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
980 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
981 if (x1 != x2 && y1 != y2)
982 break;
983 const uint *s1 = (const uint *)image.scanLine(y1);
984 const uint *s2 = (const uint *)image.scanLine(y2);
985 uint tl = s1[x1];
986 uint tr = s1[x2];
987 uint bl = s2[x1];
988 uint br = s2[x2];
989 int distx = (fx & 0x0000ffff) >> 8;
990 int disty = (fy & 0x0000ffff) >> 8;
991 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
992 fx += fdx;
993 fy += fdy;
994 ++b;
995 }
996 uint *boundedEnd = end;
997 if (fdx > 0)
998 boundedEnd = qMin(boundedEnd, b + (max_fx - fx) / fdx);
999 else if (fdx < 0)
1000 boundedEnd = qMin(boundedEnd, b + (min_fx - fx) / fdx);
1001 if (fdy > 0)
1002 boundedEnd = qMin(boundedEnd, b + (max_fy - fy) / fdy);
1003 else if (fdy < 0)
1004 boundedEnd = qMin(boundedEnd, b + (min_fy - fy) / fdy);
1005
1006 // until boundedEnd we can now have a fast middle part without boundary checks
1007 const __m256i vdistShuffle =
1008 _mm256_setr_epi8(0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80),
1009 0, char(0x80), 0, char(0x80), 4, char(0x80), 4, char(0x80), 8, char(0x80), 8, char(0x80), 12, char(0x80), 12, char(0x80));
1010 const __m256i colorMask = _mm256_set1_epi32(0x00ff00ff);
1011 const __m256i v_256 = _mm256_set1_epi16(256);
1012 const __m256i v_fdx = _mm256_set1_epi32(fdx * 8);
1013 const __m256i v_fdy = _mm256_set1_epi32(fdy * 8);
1014 const __m256i v_fxy_r = _mm256_set1_epi32(0x08);
1015 const __m256i v_index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1016 __m256i v_fx = _mm256_set1_epi32(fx);
1017 __m256i v_fy = _mm256_set1_epi32(fy);
1018 v_fx = _mm256_add_epi32(v_fx, _mm256_mullo_epi32(_mm256_set1_epi32(fdx), v_index));
1019 v_fy = _mm256_add_epi32(v_fy, _mm256_mullo_epi32(_mm256_set1_epi32(fdy), v_index));
1020
1021 const uchar *textureData = image.imageData;
1022 const qsizetype bytesPerLine = image.bytesPerLine;
1023 const __m256i vbpl = _mm256_set1_epi16(bytesPerLine/4);
1024
1025 while (b < boundedEnd - 7) {
1026 const __m256i vy = _mm256_packs_epi32(_mm256_srli_epi32(v_fy, 16), _mm256_setzero_si256());
1027 // 8x16bit * 8x16bit -> 8x32bit
1028 __m256i offset = _mm256_unpacklo_epi16(_mm256_mullo_epi16(vy, vbpl), _mm256_mulhi_epi16(vy, vbpl));
1029 offset = _mm256_add_epi32(offset, _mm256_srli_epi32(v_fx, 16));
1030 const __m128i offsetLo = _mm256_castsi256_si128(offset);
1031 const __m128i offsetHi = _mm256_extracti128_si256(offset, 1);
1032 const uint *topData = (const uint *)(textureData);
1033 const uint *botData = (const uint *)(textureData + bytesPerLine);
1034 const __m256i toplo = _mm256_i32gather_epi64((const long long *)topData, offsetLo, 4);
1035 const __m256i tophi = _mm256_i32gather_epi64((const long long *)topData, offsetHi, 4);
1036 const __m256i botlo = _mm256_i32gather_epi64((const long long *)botData, offsetLo, 4);
1037 const __m256i bothi = _mm256_i32gather_epi64((const long long *)botData, offsetHi, 4);
1038
1039 __m256i v_distx = _mm256_srli_epi16(v_fx, 8);
1040 __m256i v_disty = _mm256_srli_epi16(v_fy, 8);
1041 v_distx = _mm256_srli_epi16(_mm256_add_epi32(v_distx, v_fxy_r), 4);
1042 v_disty = _mm256_srli_epi16(_mm256_add_epi32(v_disty, v_fxy_r), 4);
1043 v_distx = _mm256_shuffle_epi8(v_distx, vdistShuffle);
1044 v_disty = _mm256_shuffle_epi8(v_disty, vdistShuffle);
1045
1046 interpolate_4_pixels_16_avx2(toplo, tophi, botlo, bothi, v_distx, v_disty, colorMask, v_256, b);
1047 b += 8;
1048 v_fx = _mm256_add_epi32(v_fx, v_fdx);
1049 v_fy = _mm256_add_epi32(v_fy, v_fdy);
1050 }
1051 fx = _mm_extract_epi32(_mm256_castsi256_si128(v_fx) , 0);
1052 fy = _mm_extract_epi32(_mm256_castsi256_si128(v_fy) , 0);
1053
1054 while (b < boundedEnd) {
1055 int x = (fx >> 16);
1056 int y = (fy >> 16);
1057
1058 const uint *s1 = (const uint *)image.scanLine(y);
1059 const uint *s2 = (const uint *)image.scanLine(y + 1);
1060
1061 int distx = (fx & 0x0000ffff) >> 8;
1062 int disty = (fy & 0x0000ffff) >> 8;
1063 *b = interpolate_4_pixels(s1 + x, s2 + x, distx, disty);
1064
1065 fx += fdx;
1066 fy += fdy;
1067 ++b;
1068 }
1069
1070 while (b < end) {
1071 int x1 = (fx >> 16);
1072 int x2;
1073 int y1 = (fy >> 16);
1074 int y2;
1075
1076 fetchTransformedBilinear_pixelBounds(image.width, image.x1, image.x2 - 1, x1, x2);
1077 fetchTransformedBilinear_pixelBounds(image.height, image.y1, image.y2 - 1, y1, y2);
1078
1079 const uint *s1 = (const uint *)image.scanLine(y1);
1080 const uint *s2 = (const uint *)image.scanLine(y2);
1081
1082 uint tl = s1[x1];
1083 uint tr = s1[x2];
1084 uint bl = s2[x1];
1085 uint br = s2[x2];
1086
1087 int distx = (fx & 0x0000ffff) >> 8;
1088 int disty = (fy & 0x0000ffff) >> 8;
1089 *b = interpolate_4_pixels(tl, tr, bl, br, distx, disty);
1090
1091 fx += fdx;
1092 fy += fdy;
1093 ++b;
1094 }
1095}
1096
1097static inline __m256i epilogueMaskFromCount(qsizetype count)
1098{
1099 Q_ASSERT(count > 0);
1100 static const __m256i offsetMask = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
1101 return _mm256_add_epi32(offsetMask, _mm256_set1_epi32(-count));
1102}
1103
1104template<bool RGBA>
1105static void convertARGBToARGB32PM_avx2(uint *buffer, const uint *src, qsizetype count)
1106{
1107 qsizetype i = 0;
1108 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1109 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1110 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1111 const __m256i half = _mm256_set1_epi16(0x0080);
1112 const __m256i zero = _mm256_setzero_si256();
1113
1114 for (; i < count - 7; i += 8) {
1115 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1116 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1117 // keep the two _mm_test[zc]_siXXX next to each other
1118 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1119 if (RGBA)
1120 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1121 if (!cf) {
1122 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1123 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1124 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1125 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1126 src1 = _mm256_mullo_epi16(src1, alpha1);
1127 src2 = _mm256_mullo_epi16(src2, alpha2);
1128 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1129 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1130 src1 = _mm256_add_epi16(src1, half);
1131 src2 = _mm256_add_epi16(src2, half);
1132 src1 = _mm256_srli_epi16(src1, 8);
1133 src2 = _mm256_srli_epi16(src2, 8);
1134 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1135 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1136 srcVector = _mm256_packus_epi16(src1, src2);
1137 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1138 } else {
1139 if (buffer != src || RGBA)
1140 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), srcVector);
1141 }
1142 } else {
1143 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), zero);
1144 }
1145 }
1146
1147 if (i < count) {
1148 const __m256i epilogueMask = epilogueMaskFromCount(count - i);
1149 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1150 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1151
1152 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1153 // keep the two _mm_test[zc]_siXXX next to each other
1154 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1155 if (RGBA)
1156 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1157 if (!cf) {
1158 __m256i src1 = _mm256_unpacklo_epi8(srcVector, zero);
1159 __m256i src2 = _mm256_unpackhi_epi8(srcVector, zero);
1160 __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1161 __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1162 src1 = _mm256_mullo_epi16(src1, alpha1);
1163 src2 = _mm256_mullo_epi16(src2, alpha2);
1164 src1 = _mm256_add_epi16(src1, _mm256_srli_epi16(src1, 8));
1165 src2 = _mm256_add_epi16(src2, _mm256_srli_epi16(src2, 8));
1166 src1 = _mm256_add_epi16(src1, half);
1167 src2 = _mm256_add_epi16(src2, half);
1168 src1 = _mm256_srli_epi16(src1, 8);
1169 src2 = _mm256_srli_epi16(src2, 8);
1170 src1 = _mm256_blend_epi16(src1, alpha1, 0x88);
1171 src2 = _mm256_blend_epi16(src2, alpha2, 0x88);
1172 srcVector = _mm256_packus_epi16(src1, src2);
1173 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1174 } else {
1175 if (buffer != src || RGBA)
1176 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, srcVector);
1177 }
1178 } else {
1179 _mm256_maskstore_epi32(reinterpret_cast<int *>(buffer + i), epilogueMask, zero);
1180 }
1181 }
1182}
1183
1184void QT_FASTCALL convertARGB32ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1185{
1186 convertARGBToARGB32PM_avx2<false>(buffer, buffer, count);
1187}
1188
1189void QT_FASTCALL convertRGBA8888ToARGB32PM_avx2(uint *buffer, int count, const QList<QRgb> *)
1190{
1191 convertARGBToARGB32PM_avx2<true>(buffer, buffer, count);
1192}
1193
1194const uint *QT_FASTCALL fetchARGB32ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1195 const QList<QRgb> *, QDitherInfo *)
1196{
1197 convertARGBToARGB32PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1198 return buffer;
1199}
1200
1201const uint *QT_FASTCALL fetchRGBA8888ToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1202 const QList<QRgb> *, QDitherInfo *)
1203{
1204 convertARGBToARGB32PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1205 return buffer;
1206}
1207
1208template<bool RGBA>
1209static void convertARGBToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, qsizetype count)
1210{
1211 qsizetype i = 0;
1212 const __m256i alphaMask = _mm256_set1_epi32(0xff000000);
1213 const __m256i rgbaMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15));
1214 const __m256i shuffleMask = _mm256_broadcastsi128_si256(_mm_setr_epi8(6, 7, 6, 7, 6, 7, 6, 7, 14, 15, 14, 15, 14, 15, 14, 15));
1215 const __m256i zero = _mm256_setzero_si256();
1216
1217 for (; i < count - 7; i += 8) {
1218 __m256i dst1, dst2;
1219 __m256i srcVector = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + i));
1220 if (!_mm256_testz_si256(srcVector, alphaMask)) {
1221 // keep the two _mm_test[zc]_siXXX next to each other
1222 bool cf = _mm256_testc_si256(srcVector, alphaMask);
1223 if (!RGBA)
1224 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1225
1226 // The two unpack instructions unpack the low and upper halves of
1227 // each 128-bit half of the 256-bit register. Here's the tracking
1228 // of what's where: (p is 32-bit, P is 64-bit)
1229 // as loaded: [ p1, p2, p3, p4; p5, p6, p7, p8 ]
1230 // after permute4x64 [ p1, p2, p5, p6; p3, p4, p7, p8 ]
1231 // after unpacklo/hi [ P1, P2; P3, P4 ] [ P5, P6; P7, P8 ]
1232 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1233
1234 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1235 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1236 if (!cf) {
1237 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1238 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1239 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1240 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1241 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1242 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1243 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1244 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1245 } else {
1246 dst1 = src1;
1247 dst2 = src2;
1248 }
1249 } else {
1250 dst1 = dst2 = zero;
1251 }
1252 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i), dst1);
1253 _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + i) + 1, dst2);
1254 }
1255
1256 if (i < count) {
1257 __m256i epilogueMask = epilogueMaskFromCount(count - i);
1258 const __m256i epilogueAlphaMask = _mm256_blendv_epi8(_mm256_setzero_si256(), alphaMask, epilogueMask);
1259 __m256i dst1, dst2;
1260 __m256i srcVector = _mm256_maskload_epi32(reinterpret_cast<const int *>(src + i), epilogueMask);
1261
1262 if (!_mm256_testz_si256(srcVector, epilogueAlphaMask)) {
1263 // keep the two _mm_test[zc]_siXXX next to each other
1264 bool cf = _mm256_testc_si256(srcVector, epilogueAlphaMask);
1265 if (!RGBA)
1266 srcVector = _mm256_shuffle_epi8(srcVector, rgbaMask);
1267 srcVector = _mm256_permute4x64_epi64(srcVector, _MM_SHUFFLE(3, 1, 2, 0));
1268 const __m256i src1 = _mm256_unpacklo_epi8(srcVector, srcVector);
1269 const __m256i src2 = _mm256_unpackhi_epi8(srcVector, srcVector);
1270 if (!cf) {
1271 const __m256i alpha1 = _mm256_shuffle_epi8(src1, shuffleMask);
1272 const __m256i alpha2 = _mm256_shuffle_epi8(src2, shuffleMask);
1273 dst1 = _mm256_mulhi_epu16(src1, alpha1);
1274 dst2 = _mm256_mulhi_epu16(src2, alpha2);
1275 dst1 = _mm256_add_epi16(dst1, _mm256_srli_epi16(dst1, 15));
1276 dst2 = _mm256_add_epi16(dst2, _mm256_srli_epi16(dst2, 15));
1277 dst1 = _mm256_blend_epi16(dst1, src1, 0x88);
1278 dst2 = _mm256_blend_epi16(dst2, src2, 0x88);
1279 } else {
1280 dst1 = src1;
1281 dst2 = src2;
1282 }
1283 } else {
1284 dst1 = dst2 = zero;
1285 }
1286 epilogueMask = _mm256_permute4x64_epi64(epilogueMask, _MM_SHUFFLE(3, 1, 2, 0));
1287 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i),
1288 _mm256_unpacklo_epi32(epilogueMask, epilogueMask),
1289 dst1);
1290 _mm256_maskstore_epi64(reinterpret_cast<qint64 *>(buffer + i + 4),
1291 _mm256_unpackhi_epi32(epilogueMask, epilogueMask),
1292 dst2);
1293 }
1294}
1295
1296const QRgba64 * QT_FASTCALL convertARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1297 const QList<QRgb> *, QDitherInfo *)
1298{
1299 convertARGBToRGBA64PM_avx2<false>(buffer, src, count);
1300 return buffer;
1301}
1302
1303const QRgba64 * QT_FASTCALL convertRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uint *src, int count,
1304 const QList<QRgb> *, QDitherInfo *)
1305{
1306 convertARGBToRGBA64PM_avx2<true>(buffer, src, count);
1307 return buffer;
1308}
1309
1310const QRgba64 *QT_FASTCALL fetchARGB32ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1311 const QList<QRgb> *, QDitherInfo *)
1312{
1313 convertARGBToRGBA64PM_avx2<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1314 return buffer;
1315}
1316
1317const QRgba64 *QT_FASTCALL fetchRGBA8888ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1318 const QList<QRgb> *, QDitherInfo *)
1319{
1320 convertARGBToRGBA64PM_avx2<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1321 return buffer;
1322}
1323
1324const QRgba64 *QT_FASTCALL fetchRGBA64ToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1325 const QList<QRgb> *, QDitherInfo *)
1326{
1327 const QRgba64 *s = reinterpret_cast<const QRgba64 *>(src) + index;
1328 int i = 0;
1329 const __m256i vh = _mm256_set1_epi32(0x8000);
1330 for (; i < count - 3; i += 4) {
1331 __m256i vs256 = _mm256_loadu_si256((const __m256i *)(s + i));
1332 __m256i va256 = _mm256_shufflelo_epi16(vs256, _MM_SHUFFLE(3, 3, 3, 3));
1333 va256 = _mm256_shufflehi_epi16(va256, _MM_SHUFFLE(3, 3, 3, 3));
1334 const __m256i vmullo = _mm256_mullo_epi16(vs256, va256);
1335 const __m256i vmulhi = _mm256_mulhi_epu16(vs256, va256);
1336 __m256i vslo = _mm256_unpacklo_epi16(vmullo, vmulhi);
1337 __m256i vshi = _mm256_unpackhi_epi16(vmullo, vmulhi);
1338 vslo = _mm256_add_epi32(vslo, _mm256_srli_epi32(vslo, 16));
1339 vshi = _mm256_add_epi32(vshi, _mm256_srli_epi32(vshi, 16));
1340 vslo = _mm256_add_epi32(vslo, vh);
1341 vshi = _mm256_add_epi32(vshi, vh);
1342 vslo = _mm256_srli_epi32(vslo, 16);
1343 vshi = _mm256_srli_epi32(vshi, 16);
1344 vs256 = _mm256_packus_epi32(vslo, vshi);
1345 vs256 = _mm256_blend_epi16(vs256, va256, 0x88);
1346 _mm256_storeu_si256((__m256i *)(buffer + i), vs256);
1347 }
1348 for (; i < count; ++i) {
1349 const auto a = s[i].alpha();
1350 __m128i vs = _mm_loadl_epi64((const __m128i *)(s + i));
1351 __m128i va = _mm_shufflelo_epi16(vs, _MM_SHUFFLE(3, 3, 3, 3));
1352 vs = multiplyAlpha65535(vs, va);
1353 _mm_storel_epi64((__m128i *)(buffer + i), vs);
1354 buffer[i].setAlpha(a);
1355 }
1356 return buffer;
1357}
1358
1359const uint *QT_FASTCALL fetchRGB16FToRGB32_avx2(uint *buffer, const uchar *src, int index, int count,
1360 const QList<QRgb> *, QDitherInfo *)
1361{
1362 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1363 const __m256 vf = _mm256_set1_ps(255.0f);
1364 const __m256 vh = _mm256_set1_ps(0.5f);
1365 int i = 0;
1366 for (; i + 1 < count; i += 2) {
1367 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1368 vsf = _mm256_mul_ps(vsf, vf);
1369 vsf = _mm256_add_ps(vsf, vh);
1370 __m256i vsi = _mm256_cvttps_epi32(vsf);
1371 vsi = _mm256_packs_epi32(vsi, vsi);
1372 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1373 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1374 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1375 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1376 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1377 }
1378 if (i < count) {
1379 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1380 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1381 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1382 __m128i vsi = _mm_cvttps_epi32(vsf);
1383 vsi = _mm_packs_epi32(vsi, vsi);
1384 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1385 vsi = _mm_packus_epi16(vsi, vsi);
1386 buffer[i] = _mm_cvtsi128_si32(vsi);
1387 }
1388 return buffer;
1389}
1390
1391const uint *QT_FASTCALL fetchRGBA16FToARGB32PM_avx2(uint *buffer, const uchar *src, int index, int count,
1392 const QList<QRgb> *, QDitherInfo *)
1393{
1394 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1395 const __m256 vf = _mm256_set1_ps(255.0f);
1396 const __m256 vh = _mm256_set1_ps(0.5f);
1397 int i = 0;
1398 for (; i + 1 < count; i += 2) {
1399 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1400 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1401 vsf = _mm256_mul_ps(vsf, vsa);
1402 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1403 vsf = _mm256_mul_ps(vsf, vf);
1404 vsf = _mm256_add_ps(vsf, vh);
1405 __m256i vsi = _mm256_cvttps_epi32(vsf);
1406 vsi = _mm256_packus_epi32(vsi, vsi);
1407 vsi = _mm256_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1408 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1409 __m128i vsi128 = _mm256_castsi256_si128(vsi);
1410 vsi128 = _mm_packus_epi16(vsi128, vsi128);
1411 _mm_storel_epi64((__m128i *)(buffer + i), vsi128);
1412 }
1413 if (i < count) {
1414 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1415 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1416 vsf = _mm_mul_ps(vsf, vsa);
1417 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1418 vsf = _mm_mul_ps(vsf, _mm_set1_ps(255.0f));
1419 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1420 __m128i vsi = _mm_cvttps_epi32(vsf);
1421 vsi = _mm_packus_epi32(vsi, vsi);
1422 vsi = _mm_shufflelo_epi16(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1423 vsi = _mm_packus_epi16(vsi, vsi);
1424 buffer[i] = _mm_cvtsi128_si32(vsi);
1425 }
1426 return buffer;
1427}
1428
1429const QRgba64 *QT_FASTCALL fetchRGBA16FPMToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1430 const QList<QRgb> *, QDitherInfo *)
1431{
1432 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1433 const __m256 vf = _mm256_set1_ps(65535.0f);
1434 const __m256 vh = _mm256_set1_ps(0.5f);
1435 int i = 0;
1436 for (; i + 1 < count; i += 2) {
1437 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1438 vsf = _mm256_mul_ps(vsf, vf);
1439 vsf = _mm256_add_ps(vsf, vh);
1440 __m256i vsi = _mm256_cvttps_epi32(vsf);
1441 vsi = _mm256_packus_epi32(vsi, vsi);
1442 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1443 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1444 }
1445 if (i < count) {
1446 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1447 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1448 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1449 __m128i vsi = _mm_cvttps_epi32(vsf);
1450 vsi = _mm_packus_epi32(vsi, vsi);
1451 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1452 }
1453 return buffer;
1454}
1455
1456const QRgba64 *QT_FASTCALL fetchRGBA16FToRGBA64PM_avx2(QRgba64 *buffer, const uchar *src, int index, int count,
1457 const QList<QRgb> *, QDitherInfo *)
1458{
1459 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1460 const __m256 vf = _mm256_set1_ps(65535.0f);
1461 const __m256 vh = _mm256_set1_ps(0.5f);
1462 int i = 0;
1463 for (; i + 1 < count; i += 2) {
1464 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1465 __m256 vsa = _mm256_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1466 vsf = _mm256_mul_ps(vsf, vsa);
1467 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1468 vsf = _mm256_mul_ps(vsf, vf);
1469 vsf = _mm256_add_ps(vsf, vh);
1470 __m256i vsi = _mm256_cvttps_epi32(vsf);
1471 vsi = _mm256_packus_epi32(vsi, vsi);
1472 vsi = _mm256_permute4x64_epi64(vsi, _MM_SHUFFLE(3, 1, 2, 0));
1473 _mm_storeu_si128((__m128i *)(buffer + i), _mm256_castsi256_si128(vsi));
1474 }
1475 if (i < count) {
1476 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1477 __m128 vsa = _mm_shuffle_ps(vsf, vsf, _MM_SHUFFLE(3, 3, 3, 3));
1478 vsf = _mm_mul_ps(vsf, vsa);
1479 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1480 vsf = _mm_mul_ps(vsf, _mm_set1_ps(65535.0f));
1481 vsf = _mm_add_ps(vsf, _mm_set1_ps(0.5f));
1482 __m128i vsi = _mm_cvttps_epi32(vsf);
1483 vsi = _mm_packus_epi32(vsi, vsi);
1484 _mm_storel_epi64((__m128i *)(buffer + i), vsi);
1485 }
1486 return buffer;
1487}
1488
1489void QT_FASTCALL storeRGB16FFromRGB32_avx2(uchar *dest, const uint *src, int index, int count,
1490 const QList<QRgb> *, QDitherInfo *)
1491{
1492 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1493 const __m256 vf = _mm256_set1_ps(1.0f / 255.0f);
1494 int i = 0;
1495 for (; i + 1 < count; i += 2) {
1496 __m256i vsi = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(src + i)));
1497 vsi = _mm256_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1498 __m256 vsf = _mm256_cvtepi32_ps(vsi);
1499 vsf = _mm256_mul_ps(vsf, vf);
1500 _mm_storeu_si128((__m128i *)(d + i), _mm256_cvtps_ph(vsf, 0));
1501 }
1502 if (i < count) {
1503 __m128i vsi = _mm_cvtsi32_si128(src[i]);
1504 vsi = _mm_cvtepu8_epi32(vsi);
1505 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1506 __m128 vsf = _mm_cvtepi32_ps(vsi);
1507 vsf = _mm_mul_ps(vsf, _mm_set1_ps(1.0f / 255.0f));
1508 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1509 }
1510}
1511
1512void QT_FASTCALL storeRGBA16FFromARGB32PM_avx2(uchar *dest, const uint *src, int index, int count,
1513 const QList<QRgb> *, QDitherInfo *)
1514{
1515 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1516 const __m128 vf = _mm_set1_ps(1.0f / 255.0f);
1517 for (int i = 0; i < count; ++i) {
1518 const uint s = src[i];
1519 __m128i vsi = _mm_cvtsi32_si128(s);
1520 vsi = _mm_cvtepu8_epi32(vsi);
1521 vsi = _mm_shuffle_epi32(vsi, _MM_SHUFFLE(3, 0, 1, 2));
1522 __m128 vsf = _mm_cvtepi32_ps(vsi);
1523 const uint8_t a = (s >> 24);
1524 if (a == 255)
1525 vsf = _mm_mul_ps(vsf, vf);
1526 else if (a == 0)
1527 vsf = _mm_set1_ps(0.0f);
1528 else {
1529 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1530 __m128 vsr = _mm_rcp_ps(vsa);
1531 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1532 vsr = _mm_insert_ps(vsr, vf, 0x30);
1533 vsf = _mm_mul_ps(vsf, vsr);
1534 }
1535 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1536 }
1537}
1538
1539#if QT_CONFIG(raster_fp)
1540const QRgbaFloat32 *QT_FASTCALL fetchRGBA16FToRGBA32F_avx2(QRgbaFloat32 *buffer, const uchar *src, int index, int count,
1541 const QList<QRgb> *, QDitherInfo *)
1542{
1543 const quint64 *s = reinterpret_cast<const quint64 *>(src) + index;
1544 int i = 0;
1545 for (; i + 1 < count; i += 2) {
1546 __m256 vsf = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(s + i)));
1547 __m256 vsa = _mm256_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1548 vsf = _mm256_mul_ps(vsf, vsa);
1549 vsf = _mm256_blend_ps(vsf, vsa, 0x88);
1550 _mm256_storeu_ps((float *)(buffer + i), vsf);
1551 }
1552 if (i < count) {
1553 __m128 vsf = _mm_cvtph_ps(_mm_loadl_epi64((const __m128i *)(s + i)));
1554 __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1555 vsf = _mm_mul_ps(vsf, vsa);
1556 vsf = _mm_insert_ps(vsf, vsa, 0x30);
1557 _mm_storeu_ps((float *)(buffer + i), vsf);
1558 }
1559 return buffer;
1560}
1561
1562void QT_FASTCALL storeRGBX16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1563 const QList<QRgb> *, QDitherInfo *)
1564{
1565 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1566 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1567 const __m128 zero = _mm_set_ps(1.0f, 0.0f, 0.0f, 0.0f);
1568 for (int i = 0; i < count; ++i) {
1569 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1570 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1571 const float a = _mm_cvtss_f32(vsa);
1572 if (a == 1.0f)
1573 { }
1574 else if (a == 0.0f)
1575 vsf = zero;
1576 else {
1577 __m128 vsr = _mm_rcp_ps(vsa);
1578 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1579 vsf = _mm_mul_ps(vsf, vsr);
1580 vsf = _mm_insert_ps(vsf, _mm_set_ss(1.0f), 0x30);
1581 }
1582 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1583 }
1584}
1585
1586void QT_FASTCALL storeRGBA16FFromRGBA32F_avx2(uchar *dest, const QRgbaFloat32 *src, int index, int count,
1587 const QList<QRgb> *, QDitherInfo *)
1588{
1589 quint64 *d = reinterpret_cast<quint64 *>(dest) + index;
1590 const __m128 *s = reinterpret_cast<const __m128 *>(src);
1591 const __m128 zero = _mm_set1_ps(0.0f);
1592 for (int i = 0; i < count; ++i) {
1593 __m128 vsf = _mm_loadu_ps(reinterpret_cast<const float *>(s + i));
1594 const __m128 vsa = _mm_permute_ps(vsf, _MM_SHUFFLE(3, 3, 3, 3));
1595 const float a = _mm_cvtss_f32(vsa);
1596 if (a == 1.0f)
1597 { }
1598 else if (a == 0.0f)
1599 vsf = zero;
1600 else {
1601 __m128 vsr = _mm_rcp_ps(vsa);
1602 vsr = _mm_sub_ps(_mm_add_ps(vsr, vsr), _mm_mul_ps(vsr, _mm_mul_ps(vsr, vsa)));
1603 vsr = _mm_insert_ps(vsr, _mm_set_ss(1.0f), 0x30);
1604 vsf = _mm_mul_ps(vsf, vsr);
1605 }
1606 _mm_storel_epi64((__m128i *)(d + i), _mm_cvtps_ph(vsf, 0));
1607 }
1608}
1609#endif
1610
1611QT_END_NAMESPACE
1612
1613#endif