Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qdrawhelper_neon.cpp
Go to the documentation of this file.
1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3// Qt-Security score:significant reason:default
4
5#include <private/qdrawhelper_neon_p.h>
6#include <private/qblendfunctions_p.h>
7#include <private/qmath_p.h>
8#include <private/qpixellayout_p.h>
9
10#ifdef __ARM_NEON__
11
12#include <private/qpaintengine_raster_p.h>
13
14QT_BEGIN_NAMESPACE
15
16void qt_memfill32(quint32 *dest, quint32 value, qsizetype count)
17{
18 const int epilogueSize = count % 16;
19 if (count >= 16) {
20 quint32 *const neonEnd = dest + count - epilogueSize;
21 const uint32x4_t valueVector1 = vdupq_n_u32(value);
22 const uint32x4x4_t valueVector4 = { valueVector1, valueVector1, valueVector1, valueVector1 };
23 do {
24 vst4q_u32(dest, valueVector4);
25 dest += 16;
26 } while (dest != neonEnd);
27 }
28
29 switch (epilogueSize)
30 {
31 case 15: *dest++ = value; Q_FALLTHROUGH();
32 case 14: *dest++ = value; Q_FALLTHROUGH();
33 case 13: *dest++ = value; Q_FALLTHROUGH();
34 case 12: *dest++ = value; Q_FALLTHROUGH();
35 case 11: *dest++ = value; Q_FALLTHROUGH();
36 case 10: *dest++ = value; Q_FALLTHROUGH();
37 case 9: *dest++ = value; Q_FALLTHROUGH();
38 case 8: *dest++ = value; Q_FALLTHROUGH();
39 case 7: *dest++ = value; Q_FALLTHROUGH();
40 case 6: *dest++ = value; Q_FALLTHROUGH();
41 case 5: *dest++ = value; Q_FALLTHROUGH();
42 case 4: *dest++ = value; Q_FALLTHROUGH();
43 case 3: *dest++ = value; Q_FALLTHROUGH();
44 case 2: *dest++ = value; Q_FALLTHROUGH();
45 case 1: *dest++ = value;
46 }
47}
48
49static inline uint16x8_t qvdiv_255_u16(uint16x8_t x, uint16x8_t half)
50{
51 // result = (x + (x >> 8) + 0x80) >> 8
52
53 const uint16x8_t temp = vshrq_n_u16(x, 8); // x >> 8
54 const uint16x8_t sum_part = vaddq_u16(x, half); // x + 0x80
55 const uint16x8_t sum = vaddq_u16(temp, sum_part);
56
57 return vshrq_n_u16(sum, 8);
58}
59
60static inline uint16x8_t qvbyte_mul_u16(uint16x8_t x, uint16x8_t alpha, uint16x8_t half)
61{
62 // t = qRound(x * alpha / 255.0)
63
64 const uint16x8_t t = vmulq_u16(x, alpha); // t
65 return qvdiv_255_u16(t, half);
66}
67
68static inline uint16x8_t qvinterpolate_pixel_255(uint16x8_t x, uint16x8_t a, uint16x8_t y, uint16x8_t b, uint16x8_t half)
69{
70 // t = x * a + y * b
71
72 const uint16x8_t ta = vmulq_u16(x, a);
73 const uint16x8_t tb = vmulq_u16(y, b);
74
75 return qvdiv_255_u16(vaddq_u16(ta, tb), half);
76}
77
78static inline uint16x8_t qvsource_over_u16(uint16x8_t src16, uint16x8_t dst16, uint16x8_t half, uint16x8_t full)
79{
80 const uint16x4_t alpha16_high = vdup_lane_u16(vget_high_u16(src16), 3);
81 const uint16x4_t alpha16_low = vdup_lane_u16(vget_low_u16(src16), 3);
82
83 const uint16x8_t alpha16 = vsubq_u16(full, vcombine_u16(alpha16_low, alpha16_high));
84
85 return vaddq_u16(src16, qvbyte_mul_u16(dst16, alpha16, half));
86}
87
88#if defined(ENABLE_PIXMAN_DRAWHELPERS)
89extern "C" void
90pixman_composite_over_8888_0565_asm_neon (int32_t w,
91 int32_t h,
92 uint16_t *dst,
93 int32_t dst_stride,
94 uint32_t *src,
95 int32_t src_stride);
96
97extern "C" void
98pixman_composite_over_8888_8888_asm_neon (int32_t w,
99 int32_t h,
100 uint32_t *dst,
101 int32_t dst_stride,
102 uint32_t *src,
103 int32_t src_stride);
104
105extern "C" void
106pixman_composite_src_0565_8888_asm_neon (int32_t w,
107 int32_t h,
108 uint32_t *dst,
109 int32_t dst_stride,
110 uint16_t *src,
111 int32_t src_stride);
112
113extern "C" void
114pixman_composite_over_n_8_0565_asm_neon (int32_t w,
115 int32_t h,
116 uint16_t *dst,
117 int32_t dst_stride,
118 uint32_t src,
119 int32_t unused,
120 uint8_t *mask,
121 int32_t mask_stride);
122
123extern "C" void
124pixman_composite_scanline_over_asm_neon (int32_t w,
125 const uint32_t *dst,
126 const uint32_t *src);
127
128extern "C" void
129pixman_composite_src_0565_0565_asm_neon (int32_t w,
130 int32_t h,
131 uint16_t *dst,
132 int32_t dst_stride,
133 uint16_t *src,
134 int32_t src_stride);
135// qblendfunctions.cpp
136void qt_blend_argb32_on_rgb16_const_alpha(uchar *destPixels, int dbpl,
137 const uchar *srcPixels, int sbpl,
138 int w, int h,
139 int const_alpha);
140
141void qt_blend_rgb16_on_argb32_neon(uchar *destPixels, int dbpl,
142 const uchar *srcPixels, int sbpl,
143 int w, int h,
144 int const_alpha)
145{
146 dbpl /= 4;
147 sbpl /= 2;
148
149 quint32 *dst = (quint32 *) destPixels;
150 quint16 *src = (quint16 *) srcPixels;
151
152 if (const_alpha != 256) {
153 quint8 a = (255 * const_alpha) >> 8;
154 quint8 ia = 255 - a;
155
156 while (--h >= 0) {
157 for (int x=0; x<w; ++x)
158 dst[x] = INTERPOLATE_PIXEL_255(qConvertRgb16To32(src[x]), a, dst[x], ia);
159 dst += dbpl;
160 src += sbpl;
161 }
162 return;
163 }
164
165 pixman_composite_src_0565_8888_asm_neon(w, h, dst, dbpl, src, sbpl);
166}
167
168// qblendfunctions.cpp
169void qt_blend_rgb16_on_rgb16(uchar *dst, int dbpl,
170 const uchar *src, int sbpl,
171 int w, int h,
172 int const_alpha);
173
174
175template <int N>
176static inline void scanLineBlit16(quint16 *dst, quint16 *src, int dstride)
177{
178 if (N >= 2) {
179 ((quint32 *)dst)[0] = ((quint32 *)src)[0];
180 __builtin_prefetch(dst + dstride, 1, 0);
181 }
182 for (int i = 1; i < N/2; ++i)
183 ((quint32 *)dst)[i] = ((quint32 *)src)[i];
184 if (N & 1)
185 dst[N-1] = src[N-1];
186}
187
188template <int Width>
189static inline void blockBlit16(quint16 *dst, quint16 *src, int dstride, int sstride, int h)
190{
191 union {
192 quintptr address;
193 quint16 *pointer;
194 } u;
195
196 u.pointer = dst;
197
198 if (u.address & 2) {
199 while (--h >= 0) {
200 // align dst
201 dst[0] = src[0];
202 if (Width > 1)
203 scanLineBlit16<Width-1>(dst + 1, src + 1, dstride);
204 dst += dstride;
205 src += sstride;
206 }
207 } else {
208 while (--h >= 0) {
209 scanLineBlit16<Width>(dst, src, dstride);
210
211 dst += dstride;
212 src += sstride;
213 }
214 }
215}
216
217void qt_blend_rgb16_on_rgb16_neon(uchar *destPixels, int dbpl,
218 const uchar *srcPixels, int sbpl,
219 int w, int h,
220 int const_alpha)
221{
222 // testing show that the default memcpy is faster for widths 150 and up
223 if (const_alpha != 256 || w >= 150) {
224 qt_blend_rgb16_on_rgb16(destPixels, dbpl, srcPixels, sbpl, w, h, const_alpha);
225 return;
226 }
227
228 int dstride = dbpl / 2;
229 int sstride = sbpl / 2;
230
231 quint16 *dst = (quint16 *) destPixels;
232 quint16 *src = (quint16 *) srcPixels;
233
234 switch (w) {
235#define BLOCKBLIT(n) case n: blockBlit16<n>(dst, src, dstride, sstride, h); return;
236 BLOCKBLIT(1);
237 BLOCKBLIT(2);
238 BLOCKBLIT(3);
239 BLOCKBLIT(4);
240 BLOCKBLIT(5);
241 BLOCKBLIT(6);
242 BLOCKBLIT(7);
243 BLOCKBLIT(8);
244 BLOCKBLIT(9);
245 BLOCKBLIT(10);
246 BLOCKBLIT(11);
247 BLOCKBLIT(12);
248 BLOCKBLIT(13);
249 BLOCKBLIT(14);
250 BLOCKBLIT(15);
251#undef BLOCKBLIT
252 default:
253 break;
254 }
255
256 pixman_composite_src_0565_0565_asm_neon (w, h, dst, dstride, src, sstride);
257}
258
259extern "C" void blend_8_pixels_argb32_on_rgb16_neon(quint16 *dst, const quint32 *src, int const_alpha);
260
261void qt_blend_argb32_on_rgb16_neon(uchar *destPixels, int dbpl,
262 const uchar *srcPixels, int sbpl,
263 int w, int h,
264 int const_alpha)
265{
266 quint16 *dst = (quint16 *) destPixels;
267 quint32 *src = (quint32 *) srcPixels;
268
269 if (const_alpha != 256) {
270 for (int y=0; y<h; ++y) {
271 int i = 0;
272 for (; i < w-7; i += 8)
273 blend_8_pixels_argb32_on_rgb16_neon(&dst[i], &src[i], const_alpha);
274
275 if (i < w) {
276 int tail = w - i;
277
278 quint16 dstBuffer[8];
279 quint32 srcBuffer[8];
280
281 for (int j = 0; j < tail; ++j) {
282 dstBuffer[j] = dst[i + j];
283 srcBuffer[j] = src[i + j];
284 }
285
286 blend_8_pixels_argb32_on_rgb16_neon(dstBuffer, srcBuffer, const_alpha);
287
288 for (int j = 0; j < tail; ++j)
289 dst[i + j] = dstBuffer[j];
290 }
291
292 dst = (quint16 *)(((uchar *) dst) + dbpl);
293 src = (quint32 *)(((uchar *) src) + sbpl);
294 }
295 return;
296 }
297
298 pixman_composite_over_8888_0565_asm_neon(w, h, dst, dbpl / 2, src, sbpl / 4);
299}
300#endif
301
302inline void qt_blend_argb32_on_argb32_opaque_neon(uchar *destPixels, int dbpl,
303 const uchar *srcPixels, int sbpl,
304 int w, int h)
305{
306#if defined(ENABLE_PIXMAN_DRAWHELPERS)
307 pixman_composite_over_8888_8888_asm_neon(w, h, (uint32_t *)destPixels, dbpl / 4, (uint32_t *)srcPixels, sbpl / 4);
308#else
309 uint32_t *dst = reinterpret_cast<uint32_t *>(destPixels);
310 const uint32_t *src = reinterpret_cast<const uint32_t *>(srcPixels);
311 const uint16x8_t half = vdupq_n_u16(0x80);
312 const uint16x8_t full = vdupq_n_u16(0xff);
313 for (int y = 0; y < h; ++y) {
314 int x = 0;
315 for (; x < (w - 3); x += 4) {
316 if ((src[x] | src[x + 1] | src[x + 2] | src[x + 3]) != 0) {
317 uint32x4_t src32 = vld1q_u32(src + x);
318 uint32x4_t dst32 = vld1q_u32(dst + x);
319
320 const uint8x16_t src8 = vreinterpretq_u8_u32(src32);
321 const uint8x16_t dst8 = vreinterpretq_u8_u32(dst32);
322
323 const uint8x8_t src8_low = vget_low_u8(src8);
324 const uint8x8_t dst8_low = vget_low_u8(dst8);
325
326 const uint8x8_t src8_high = vget_high_u8(src8);
327 const uint8x8_t dst8_high = vget_high_u8(dst8);
328
329 const uint16x8_t src16_low = vmovl_u8(src8_low);
330 const uint16x8_t dst16_low = vmovl_u8(dst8_low);
331
332 const uint16x8_t src16_high = vmovl_u8(src8_high);
333 const uint16x8_t dst16_high = vmovl_u8(dst8_high);
334
335 const uint16x8_t result16_low = qvsource_over_u16(src16_low, dst16_low, half, full);
336 const uint16x8_t result16_high = qvsource_over_u16(src16_high, dst16_high, half, full);
337
338 const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result16_low));
339 const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result16_high));
340
341 vst1q_u32(dst + x, vcombine_u32(result32_low, result32_high));
342 }
343 }
344 for (; x < w; ++x) {
345 const uint s = src[x];
346 if (s >= 0xff000000)
347 dst[x] = s;
348 else if (s != 0)
349 dst[x] = s + BYTE_MUL(dst[x], qAlpha(~s));
350 }
351 destPixels += dbpl;
352 srcPixels += sbpl;
353 dst = reinterpret_cast<uint32_t *>(destPixels);
354 src = reinterpret_cast<const uint32_t *>(srcPixels);
355 }
356#endif
357}
358
359inline void qt_blend_argb32_on_argb32_with_alpha_neon(uchar *destPixels, int dbpl,
360 const uchar *srcPixels, int sbpl,
361 int w, int h,
362 uint const_alpha)
363{
364 Q_ASSERT(const_alpha > 0 && const_alpha < 256); // const_alpha on ]0-255] form
365 uint32_t *dst = reinterpret_cast<uint32_t *>(destPixels);
366 const uint32_t *src = reinterpret_cast<const uint32_t *>(srcPixels);
367 const uint16x8_t half = vdupq_n_u16(0x80);
368 const uint16x8_t full = vdupq_n_u16(0xff);
369 const uint16x8_t const_alpha16 = vdupq_n_u16(const_alpha);
370 for (int y = 0; y < h; ++y) {
371 int x = 0;
372 for (; x < (w - 3); x += 4) {
373 if ((src[x] | src[x + 1] | src[x + 2] | src[x + 3]) != 0) {
374 uint32x4_t src32 = vld1q_u32(src + x);
375 uint32x4_t dst32 = vld1q_u32(dst + x);
376
377 const uint8x16_t src8 = vreinterpretq_u8_u32(src32);
378 const uint8x16_t dst8 = vreinterpretq_u8_u32(dst32);
379
380 const uint8x8_t src8_low = vget_low_u8(src8);
381 const uint8x8_t dst8_low = vget_low_u8(dst8);
382
383 const uint8x8_t src8_high = vget_high_u8(src8);
384 const uint8x8_t dst8_high = vget_high_u8(dst8);
385
386 const uint16x8_t src16_low = vmovl_u8(src8_low);
387 const uint16x8_t dst16_low = vmovl_u8(dst8_low);
388
389 const uint16x8_t src16_high = vmovl_u8(src8_high);
390 const uint16x8_t dst16_high = vmovl_u8(dst8_high);
391
392 const uint16x8_t srcalpha16_low = qvbyte_mul_u16(src16_low, const_alpha16, half);
393 const uint16x8_t srcalpha16_high = qvbyte_mul_u16(src16_high, const_alpha16, half);
394
395 const uint16x8_t result16_low = qvsource_over_u16(srcalpha16_low, dst16_low, half, full);
396 const uint16x8_t result16_high = qvsource_over_u16(srcalpha16_high, dst16_high, half, full);
397
398 const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result16_low));
399 const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result16_high));
400
401 vst1q_u32(dst + x, vcombine_u32(result32_low, result32_high));
402 }
403 }
404 for (; x < w; ++x) {
405 uint s = src[x];
406 if (s != 0) {
407 s = BYTE_MUL(s, const_alpha);
408 dst[x] = s + BYTE_MUL(dst[x], qAlpha(~s));
409 }
410 }
411 destPixels += dbpl;
412 srcPixels += sbpl;
413 dst = reinterpret_cast<uint32_t *>(destPixels);
414 src = reinterpret_cast<const uint32_t *>(srcPixels);
415 }
416}
417
418void comp_func_SourceOver_neon(uint *dest, const uint *src, int length, uint const_alpha)
419{
420 Q_ASSERT(const_alpha < 256); // const_alpha on [0-255] form
421 if (const_alpha == 255) {
422#if defined(ENABLE_PIXMAN_DRAWHELPERS)
423 pixman_composite_scanline_over_asm_neon(length, dest, src);
424#else
425 qt_blend_argb32_on_argb32_opaque_neon(reinterpret_cast<uchar *>(dest), 0, reinterpret_cast<const uchar *>(src), 0, length, 1);
426#endif
427 } else if (const_alpha != 0) {
428 qt_blend_argb32_on_argb32_with_alpha_neon(reinterpret_cast<uchar *>(dest), 0, reinterpret_cast<const uchar *>(src), 0, length, 1, const_alpha);
429 }
430}
431
432void qt_blend_argb32_on_argb32_neon(uchar *destPixels, int dbpl,
433 const uchar *srcPixels, int sbpl,
434 int w, int h,
435 int const_alpha)
436{
437 Q_ASSERT(const_alpha >= 0 && const_alpha <= 256); // const_alpha on [0-256] form
438 if (const_alpha == 256) {
439 qt_blend_argb32_on_argb32_opaque_neon(destPixels, dbpl,
440 srcPixels, sbpl,
441 w, h);
442 } else if (const_alpha != 0) {
443 const uint const_alpha_255 = (static_cast<uint>(const_alpha) * 255) >> 8;
444 qt_blend_argb32_on_argb32_with_alpha_neon(destPixels, dbpl,
445 srcPixels, sbpl,
446 w, h,
447 const_alpha_255);
448 }
449}
450
451// qblendfunctions.cpp
452void qt_blend_rgb32_on_rgb32(uchar *destPixels, int dbpl,
453 const uchar *srcPixels, int sbpl,
454 int w, int h,
455 int const_alpha);
456
457void qt_blend_rgb32_on_rgb32_neon(uchar *destPixels, int dbpl,
458 const uchar *srcPixels, int sbpl,
459 int w, int h,
460 int const_alpha)
461{
462 if (const_alpha != 256) {
463 if (const_alpha != 0) {
464 const uint *src = (const uint *) srcPixels;
465 uint *dst = (uint *) destPixels;
466 uint16x8_t half = vdupq_n_u16(0x80);
467 const_alpha = (const_alpha * 255) >> 8;
468 int one_minus_const_alpha = 255 - const_alpha;
469 uint16x8_t const_alpha16 = vdupq_n_u16(const_alpha);
470 uint16x8_t one_minus_const_alpha16 = vdupq_n_u16(255 - const_alpha);
471 for (int y = 0; y < h; ++y) {
472 int x = 0;
473 for (; x < w-3; x += 4) {
474 uint32x4_t src32 = vld1q_u32((uint32_t *)&src[x]);
475 uint32x4_t dst32 = vld1q_u32((uint32_t *)&dst[x]);
476
477 const uint8x16_t src8 = vreinterpretq_u8_u32(src32);
478 const uint8x16_t dst8 = vreinterpretq_u8_u32(dst32);
479
480 const uint8x8_t src8_low = vget_low_u8(src8);
481 const uint8x8_t dst8_low = vget_low_u8(dst8);
482
483 const uint8x8_t src8_high = vget_high_u8(src8);
484 const uint8x8_t dst8_high = vget_high_u8(dst8);
485
486 const uint16x8_t src16_low = vmovl_u8(src8_low);
487 const uint16x8_t dst16_low = vmovl_u8(dst8_low);
488
489 const uint16x8_t src16_high = vmovl_u8(src8_high);
490 const uint16x8_t dst16_high = vmovl_u8(dst8_high);
491
492 const uint16x8_t result16_low = qvinterpolate_pixel_255(src16_low, const_alpha16, dst16_low, one_minus_const_alpha16, half);
493 const uint16x8_t result16_high = qvinterpolate_pixel_255(src16_high, const_alpha16, dst16_high, one_minus_const_alpha16, half);
494
495 const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result16_low));
496 const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result16_high));
497
498 vst1q_u32((uint32_t *)&dst[x], vcombine_u32(result32_low, result32_high));
499 }
500 for (; x<w; ++x) {
501 dst[x] = INTERPOLATE_PIXEL_255(src[x], const_alpha, dst[x], one_minus_const_alpha);
502 }
503 dst = (quint32 *)(((uchar *) dst) + dbpl);
504 src = (const quint32 *)(((const uchar *) src) + sbpl);
505 }
506 }
507 } else {
508 qt_blend_rgb32_on_rgb32(destPixels, dbpl, srcPixels, sbpl, w, h, const_alpha);
509 }
510}
511
512#if defined(ENABLE_PIXMAN_DRAWHELPERS)
513extern void qt_alphamapblit_quint16(QRasterBuffer *rasterBuffer,
514 int x, int y, const QRgba64 &color,
515 const uchar *map,
516 int mapWidth, int mapHeight, int mapStride,
517 const QClipData *clip, bool useGammaCorrection);
518
519void qt_alphamapblit_quint16_neon(QRasterBuffer *rasterBuffer,
520 int x, int y, const QRgba64 &color,
521 const uchar *bitmap,
522 int mapWidth, int mapHeight, int mapStride,
523 const QClipData *clip, bool useGammaCorrection)
524{
525 if (clip || useGammaCorrection) {
526 qt_alphamapblit_quint16(rasterBuffer, x, y, color, bitmap, mapWidth, mapHeight, mapStride, clip, useGammaCorrection);
527 return;
528 }
529
530 quint16 *dest = reinterpret_cast<quint16*>(rasterBuffer->scanLine(y)) + x;
531 const int destStride = rasterBuffer->bytesPerLine() / sizeof(quint16);
532
533 uchar *mask = const_cast<uchar *>(bitmap);
534 const uint c = color.toArgb32();
535
536 pixman_composite_over_n_8_0565_asm_neon(mapWidth, mapHeight, dest, destStride, c, 0, mask, mapStride);
537}
538
539extern "C" void blend_8_pixels_rgb16_on_rgb16_neon(quint16 *dst, const quint16 *src, int const_alpha);
540
541template <typename SRC, typename BlendFunc>
542struct Blend_on_RGB16_SourceAndConstAlpha_Neon {
543 Blend_on_RGB16_SourceAndConstAlpha_Neon(BlendFunc blender, int const_alpha)
544 : m_index(0)
545 , m_blender(blender)
546 , m_const_alpha(const_alpha)
547 {
548 }
549
550 inline void write(quint16 *dst, quint32 src)
551 {
552 srcBuffer[m_index++] = src;
553
554 if (m_index == 8) {
555 m_blender(dst - 7, srcBuffer, m_const_alpha);
556 m_index = 0;
557 }
558 }
559
560 inline void flush(quint16 *dst)
561 {
562 if (m_index > 0) {
563 quint16 dstBuffer[8];
564 for (int i = 0; i < m_index; ++i)
565 dstBuffer[i] = dst[i - m_index];
566
567 m_blender(dstBuffer, srcBuffer, m_const_alpha);
568
569 for (int i = 0; i < m_index; ++i)
570 dst[i - m_index] = dstBuffer[i];
571
572 m_index = 0;
573 }
574 }
575
576 SRC srcBuffer[8];
577
578 int m_index;
579 BlendFunc m_blender;
580 int m_const_alpha;
581};
582
583template <typename SRC, typename BlendFunc>
584Blend_on_RGB16_SourceAndConstAlpha_Neon<SRC, BlendFunc>
585Blend_on_RGB16_SourceAndConstAlpha_Neon_create(BlendFunc blender, int const_alpha)
586{
587 return Blend_on_RGB16_SourceAndConstAlpha_Neon<SRC, BlendFunc>(blender, const_alpha);
588}
589
590void qt_scale_image_argb32_on_rgb16_neon(uchar *destPixels, int dbpl,
591 const uchar *srcPixels, int sbpl, int srch,
592 const QRectF &targetRect,
593 const QRectF &sourceRect,
594 const QRect &clip,
595 int const_alpha)
596{
597 if (const_alpha == 0)
598 return;
599
600 qt_scale_image_16bit<quint32>(destPixels, dbpl, srcPixels, sbpl, srch, targetRect, sourceRect, clip,
601 Blend_on_RGB16_SourceAndConstAlpha_Neon_create<quint32>(blend_8_pixels_argb32_on_rgb16_neon, const_alpha));
602}
603
604void qt_scale_image_rgb16_on_rgb16(uchar *destPixels, int dbpl,
605 const uchar *srcPixels, int sbpl, int srch,
606 const QRectF &targetRect,
607 const QRectF &sourceRect,
608 const QRect &clip,
609 int const_alpha);
610
611void qt_scale_image_rgb16_on_rgb16_neon(uchar *destPixels, int dbpl,
612 const uchar *srcPixels, int sbpl, int srch,
613 const QRectF &targetRect,
614 const QRectF &sourceRect,
615 const QRect &clip,
616 int const_alpha)
617{
618 if (const_alpha == 0)
619 return;
620
621 if (const_alpha == 256) {
622 qt_scale_image_rgb16_on_rgb16(destPixels, dbpl, srcPixels, sbpl, srch, targetRect, sourceRect, clip, const_alpha);
623 return;
624 }
625
626 qt_scale_image_16bit<quint16>(destPixels, dbpl, srcPixels, sbpl, srch, targetRect, sourceRect, clip,
627 Blend_on_RGB16_SourceAndConstAlpha_Neon_create<quint16>(blend_8_pixels_rgb16_on_rgb16_neon, const_alpha));
628}
629
630extern void qt_transform_image_rgb16_on_rgb16(uchar *destPixels, int dbpl,
631 const uchar *srcPixels, int sbpl,
632 const QRectF &targetRect,
633 const QRectF &sourceRect,
634 const QRect &clip,
635 const QTransform &targetRectTransform,
636 int const_alpha);
637
638void qt_transform_image_rgb16_on_rgb16_neon(uchar *destPixels, int dbpl,
639 const uchar *srcPixels, int sbpl,
640 const QRectF &targetRect,
641 const QRectF &sourceRect,
642 const QRect &clip,
643 const QTransform &targetRectTransform,
644 int const_alpha)
645{
646 if (const_alpha == 0)
647 return;
648
649 if (const_alpha == 256) {
650 qt_transform_image_rgb16_on_rgb16(destPixels, dbpl, srcPixels, sbpl, targetRect, sourceRect, clip, targetRectTransform, const_alpha);
651 return;
652 }
653
654 qt_transform_image(reinterpret_cast<quint16 *>(destPixels), dbpl,
655 reinterpret_cast<const quint16 *>(srcPixels), sbpl, targetRect, sourceRect, clip, targetRectTransform,
656 Blend_on_RGB16_SourceAndConstAlpha_Neon_create<quint16>(blend_8_pixels_rgb16_on_rgb16_neon, const_alpha));
657}
658
659void qt_transform_image_argb32_on_rgb16_neon(uchar *destPixels, int dbpl,
660 const uchar *srcPixels, int sbpl,
661 const QRectF &targetRect,
662 const QRectF &sourceRect,
663 const QRect &clip,
664 const QTransform &targetRectTransform,
665 int const_alpha)
666{
667 if (const_alpha == 0)
668 return;
669
670 qt_transform_image(reinterpret_cast<quint16 *>(destPixels), dbpl,
671 reinterpret_cast<const quint32 *>(srcPixels), sbpl, targetRect, sourceRect, clip, targetRectTransform,
672 Blend_on_RGB16_SourceAndConstAlpha_Neon_create<quint32>(blend_8_pixels_argb32_on_rgb16_neon, const_alpha));
673}
674
675static inline void convert_8_pixels_rgb16_to_argb32(quint32 *dst, const quint16 *src)
676{
677 asm volatile (
678 "vld1.16 { d0, d1 }, [%[SRC]]\n\t"
679
680 /* convert 8 r5g6b5 pixel data from {d0, d1} to planar 8-bit format
681 and put data into d4 - red, d3 - green, d2 - blue */
682 "vshrn.u16 d4, q0, #8\n\t"
683 "vshrn.u16 d3, q0, #3\n\t"
684 "vsli.u16 q0, q0, #5\n\t"
685 "vsri.u8 d4, d4, #5\n\t"
686 "vsri.u8 d3, d3, #6\n\t"
687 "vshrn.u16 d2, q0, #2\n\t"
688
689 /* fill d5 - alpha with 0xff */
690 "mov r2, #255\n\t"
691 "vdup.8 d5, r2\n\t"
692
693 "vst4.8 { d2, d3, d4, d5 }, [%[DST]]"
694 : : [DST]"r" (dst), [SRC]"r" (src)
695 : "memory", "r2", "d0", "d1", "d2", "d3", "d4", "d5"
696 );
697}
698
699uint * QT_FASTCALL qt_destFetchRGB16_neon(uint *buffer, QRasterBuffer *rasterBuffer, int x, int y, int length)
700{
701 const ushort *data = (const ushort *)rasterBuffer->scanLine(y) + x;
702
703 int i = 0;
704 for (; i < length - 7; i += 8)
705 convert_8_pixels_rgb16_to_argb32(&buffer[i], &data[i]);
706
707 if (i < length) {
708 quint16 srcBuffer[8];
709 quint32 dstBuffer[8];
710
711 int tail = length - i;
712 for (int j = 0; j < tail; ++j)
713 srcBuffer[j] = data[i + j];
714
715 convert_8_pixels_rgb16_to_argb32(dstBuffer, srcBuffer);
716
717 for (int j = 0; j < tail; ++j)
718 buffer[i + j] = dstBuffer[j];
719 }
720
721 return buffer;
722}
723
724static inline void convert_8_pixels_argb32_to_rgb16(quint16 *dst, const quint32 *src)
725{
726 asm volatile (
727 "vld4.8 { d0, d1, d2, d3 }, [%[SRC]]\n\t"
728
729 /* convert to r5g6b5 and store it into {d28, d29} */
730 "vshll.u8 q14, d2, #8\n\t"
731 "vshll.u8 q8, d1, #8\n\t"
732 "vshll.u8 q9, d0, #8\n\t"
733 "vsri.u16 q14, q8, #5\n\t"
734 "vsri.u16 q14, q9, #11\n\t"
735
736 "vst1.16 { d28, d29 }, [%[DST]]"
737 : : [DST]"r" (dst), [SRC]"r" (src)
738 : "memory", "d0", "d1", "d2", "d3", "d16", "d17", "d18", "d19", "d28", "d29"
739 );
740}
741
742void QT_FASTCALL qt_destStoreRGB16_neon(QRasterBuffer *rasterBuffer, int x, int y, const uint *buffer, int length)
743{
744 quint16 *data = (quint16*)rasterBuffer->scanLine(y) + x;
745
746 int i = 0;
747 for (; i < length - 7; i += 8)
748 convert_8_pixels_argb32_to_rgb16(&data[i], &buffer[i]);
749
750 if (i < length) {
751 quint32 srcBuffer[8];
752 quint16 dstBuffer[8];
753
754 int tail = length - i;
755 for (int j = 0; j < tail; ++j)
756 srcBuffer[j] = buffer[i + j];
757
758 convert_8_pixels_argb32_to_rgb16(dstBuffer, srcBuffer);
759
760 for (int j = 0; j < tail; ++j)
761 data[i + j] = dstBuffer[j];
762 }
763}
764#endif
765
766void QT_FASTCALL comp_func_solid_SourceOver_neon(uint *destPixels, int length, uint color, uint const_alpha)
767{
768 if ((const_alpha & qAlpha(color)) == 255) {
769 qt_memfill32(destPixels, color, length);
770 } else {
771 if (const_alpha != 255)
772 color = BYTE_MUL(color, const_alpha);
773
774 const quint32 minusAlphaOfColor = qAlpha(~color);
775 int x = 0;
776
777 uint32_t *dst = (uint32_t *) destPixels;
778 const uint32x4_t colorVector = vdupq_n_u32(color);
779 uint16x8_t half = vdupq_n_u16(0x80);
780 const uint16x8_t minusAlphaOfColorVector = vdupq_n_u16(minusAlphaOfColor);
781
782 for (; x < length-3; x += 4) {
783 uint32x4_t dstVector = vld1q_u32(&dst[x]);
784
785 const uint8x16_t dst8 = vreinterpretq_u8_u32(dstVector);
786
787 const uint8x8_t dst8_low = vget_low_u8(dst8);
788 const uint8x8_t dst8_high = vget_high_u8(dst8);
789
790 const uint16x8_t dst16_low = vmovl_u8(dst8_low);
791 const uint16x8_t dst16_high = vmovl_u8(dst8_high);
792
793 const uint16x8_t result16_low = qvbyte_mul_u16(dst16_low, minusAlphaOfColorVector, half);
794 const uint16x8_t result16_high = qvbyte_mul_u16(dst16_high, minusAlphaOfColorVector, half);
795
796 const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result16_low));
797 const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result16_high));
798
799 uint32x4_t blendedPixels = vcombine_u32(result32_low, result32_high);
800 uint32x4_t colorPlusBlendedPixels = vaddq_u32(colorVector, blendedPixels);
801 vst1q_u32(&dst[x], colorPlusBlendedPixels);
802 }
803
804 SIMD_EPILOGUE(x, length, 3)
805 destPixels[x] = color + BYTE_MUL(destPixels[x], minusAlphaOfColor);
806 }
807}
808
809void QT_FASTCALL comp_func_Plus_neon(uint *dst, const uint *src, int length, uint const_alpha)
810{
811 if (const_alpha == 255) {
812 uint *const end = dst + length;
813 uint *const neonEnd = end - 3;
814
815 while (dst < neonEnd) {
816 uint8x16_t vs = vld1q_u8((const uint8_t*)src);
817 const uint8x16_t vd = vld1q_u8((uint8_t*)dst);
818 vs = vqaddq_u8(vs, vd);
819 vst1q_u8((uint8_t*)dst, vs);
820 src += 4;
821 dst += 4;
822 };
823
824 while (dst != end) {
825 *dst = comp_func_Plus_one_pixel(*dst, *src);
826 ++dst;
827 ++src;
828 }
829 } else {
830 int x = 0;
831 const int one_minus_const_alpha = 255 - const_alpha;
832 const uint16x8_t constAlphaVector = vdupq_n_u16(const_alpha);
833 const uint16x8_t oneMinusconstAlphaVector = vdupq_n_u16(one_minus_const_alpha);
834
835 const uint16x8_t half = vdupq_n_u16(0x80);
836 for (; x < length - 3; x += 4) {
837 const uint32x4_t src32 = vld1q_u32((uint32_t *)&src[x]);
838 const uint8x16_t src8 = vreinterpretq_u8_u32(src32);
839 uint8x16_t dst8 = vld1q_u8((uint8_t *)&dst[x]);
840 uint8x16_t result = vqaddq_u8(dst8, src8);
841
842 uint16x8_t result_low = vmovl_u8(vget_low_u8(result));
843 uint16x8_t result_high = vmovl_u8(vget_high_u8(result));
844
845 uint16x8_t dst_low = vmovl_u8(vget_low_u8(dst8));
846 uint16x8_t dst_high = vmovl_u8(vget_high_u8(dst8));
847
848 result_low = qvinterpolate_pixel_255(result_low, constAlphaVector, dst_low, oneMinusconstAlphaVector, half);
849 result_high = qvinterpolate_pixel_255(result_high, constAlphaVector, dst_high, oneMinusconstAlphaVector, half);
850
851 const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result_low));
852 const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result_high));
853 vst1q_u32((uint32_t *)&dst[x], vcombine_u32(result32_low, result32_high));
854 }
855
856 SIMD_EPILOGUE(x, length, 3)
857 dst[x] = comp_func_Plus_one_pixel_const_alpha(dst[x], src[x], const_alpha, one_minus_const_alpha);
858 }
859}
860
861#if defined(ENABLE_PIXMAN_DRAWHELPERS)
862static const int tileSize = 32;
863
864extern "C" void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count);
865
866void qt_memrotate90_16_neon(const uchar *srcPixels, int w, int h, int sstride, uchar *destPixels, int dstride)
867{
868 const ushort *src = (const ushort *)srcPixels;
869 ushort *dest = (ushort *)destPixels;
870
871 sstride /= sizeof(ushort);
872 dstride /= sizeof(ushort);
873
874 const int pack = sizeof(quint32) / sizeof(ushort);
875 const int unaligned =
876 qMin(uint((quintptr(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h));
877 const int restX = w % tileSize;
878 const int restY = (h - unaligned) % tileSize;
879 const int unoptimizedY = restY % pack;
880 const int numTilesX = w / tileSize + (restX > 0);
881 const int numTilesY = (h - unaligned) / tileSize + (restY >= pack);
882
883 for (int tx = 0; tx < numTilesX; ++tx) {
884 const int startx = w - tx * tileSize - 1;
885 const int stopx = qMax(startx - tileSize, 0);
886
887 if (unaligned) {
888 for (int x = startx; x >= stopx; --x) {
889 ushort *d = dest + (w - x - 1) * dstride;
890 for (int y = 0; y < unaligned; ++y) {
891 *d++ = src[y * sstride + x];
892 }
893 }
894 }
895
896 for (int ty = 0; ty < numTilesY; ++ty) {
897 const int starty = ty * tileSize + unaligned;
898 const int stopy = qMin(starty + tileSize, h - unoptimizedY);
899
900 int x = startx;
901 // qt_rotate90_16_neon writes to eight rows, four pixels at a time
902 for (; x >= stopx + 7; x -= 8) {
903 ushort *d = dest + (w - x - 1) * dstride + starty;
904 const ushort *s = &src[starty * sstride + x - 7];
905 qt_rotate90_16_neon(d, s, sstride * 2, dstride * 2, stopy - starty);
906 }
907
908 for (; x >= stopx; --x) {
909 quint32 *d = reinterpret_cast<quint32*>(dest + (w - x - 1) * dstride + starty);
910 for (int y = starty; y < stopy; y += pack) {
911 quint32 c = src[y * sstride + x];
912 for (int i = 1; i < pack; ++i) {
913 const int shift = (sizeof(int) * 8 / pack * i);
914 const ushort color = src[(y + i) * sstride + x];
915 c |= color << shift;
916 }
917 *d++ = c;
918 }
919 }
920 }
921
922 if (unoptimizedY) {
923 const int starty = h - unoptimizedY;
924 for (int x = startx; x >= stopx; --x) {
925 ushort *d = dest + (w - x - 1) * dstride + starty;
926 for (int y = starty; y < h; ++y) {
927 *d++ = src[y * sstride + x];
928 }
929 }
930 }
931 }
932}
933
934extern "C" void qt_rotate270_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count);
935
936void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h,
937 int sstride,
938 uchar *destPixels, int dstride)
939{
940 const ushort *src = (const ushort *)srcPixels;
941 ushort *dest = (ushort *)destPixels;
942
943 sstride /= sizeof(ushort);
944 dstride /= sizeof(ushort);
945
946 const int pack = sizeof(quint32) / sizeof(ushort);
947 const int unaligned =
948 qMin(uint((long(dest) & (sizeof(quint32)-1)) / sizeof(ushort)), uint(h));
949 const int restX = w % tileSize;
950 const int restY = (h - unaligned) % tileSize;
951 const int unoptimizedY = restY % pack;
952 const int numTilesX = w / tileSize + (restX > 0);
953 const int numTilesY = (h - unaligned) / tileSize + (restY >= pack);
954
955 for (int tx = 0; tx < numTilesX; ++tx) {
956 const int startx = tx * tileSize;
957 const int stopx = qMin(startx + tileSize, w);
958
959 if (unaligned) {
960 for (int x = startx; x < stopx; ++x) {
961 ushort *d = dest + x * dstride;
962 for (int y = h - 1; y >= h - unaligned; --y) {
963 *d++ = src[y * sstride + x];
964 }
965 }
966 }
967
968 for (int ty = 0; ty < numTilesY; ++ty) {
969 const int starty = h - 1 - unaligned - ty * tileSize;
970 const int stopy = qMax(starty - tileSize, unoptimizedY);
971
972 int x = startx;
973 // qt_rotate90_16_neon writes to eight rows, four pixels at a time
974 for (; x < stopx - 7; x += 8) {
975 ushort *d = dest + x * dstride + h - 1 - starty;
976 const ushort *s = &src[starty * sstride + x];
977 qt_rotate90_16_neon(d + 7 * dstride, s, -sstride * 2, -dstride * 2, starty - stopy);
978 }
979
980 for (; x < stopx; ++x) {
981 quint32 *d = reinterpret_cast<quint32*>(dest + x * dstride
982 + h - 1 - starty);
983 for (int y = starty; y > stopy; y -= pack) {
984 quint32 c = src[y * sstride + x];
985 for (int i = 1; i < pack; ++i) {
986 const int shift = (sizeof(int) * 8 / pack * i);
987 const ushort color = src[(y - i) * sstride + x];
988 c |= color << shift;
989 }
990 *d++ = c;
991 }
992 }
993 }
994 if (unoptimizedY) {
995 const int starty = unoptimizedY - 1;
996 for (int x = startx; x < stopx; ++x) {
997 ushort *d = dest + x * dstride + h - 1 - starty;
998 for (int y = starty; y >= 0; --y) {
999 *d++ = src[y * sstride + x];
1000 }
1001 }
1002 }
1003 }
1004}
1005#endif
1006
1007class QSimdNeon
1008{
1009public:
1010 struct Int32x4 {
1011 Int32x4() = default;
1012 Int32x4(int32x4_t v) : v(v) {}
1013 int32x4_t v;
1014 operator int32x4_t() const { return v; }
1015 };
1016 struct Float32x4 {
1017 Float32x4() = default;
1018 Float32x4(float32x4_t v) : v(v) {};
1019 float32x4_t v;
1020 operator float32x4_t() const { return v; }
1021 };
1022
1023 union Vect_buffer_i { Int32x4 v; int i[4]; };
1024 union Vect_buffer_f { Float32x4 v; float f[4]; };
1025
1026 static inline Float32x4 v_dup(double x) { return vdupq_n_f32(float(x)); }
1027 static inline Float32x4 v_dup(float x) { return vdupq_n_f32(x); }
1028 static inline Int32x4 v_dup(int x) { return vdupq_n_s32(x); }
1029 static inline Int32x4 v_dup(uint x) { return vdupq_n_s32(x); }
1030
1031 static inline Float32x4 v_add(Float32x4 a, Float32x4 b) { return vaddq_f32(a, b); }
1032 static inline Int32x4 v_add(Int32x4 a, Int32x4 b) { return vaddq_s32(a, b); }
1033
1034 static inline Float32x4 v_max(Float32x4 a, Float32x4 b) { return vmaxq_f32(a, b); }
1035 static inline Float32x4 v_min(Float32x4 a, Float32x4 b) { return vminq_f32(a, b); }
1036 static inline Int32x4 v_min_16(Int32x4 a, Int32x4 b) { return vminq_s32(a, b); }
1037
1038 static inline Int32x4 v_and(Int32x4 a, Int32x4 b) { return vandq_s32(a, b); }
1039
1040 static inline Float32x4 v_sub(Float32x4 a, Float32x4 b) { return vsubq_f32(a, b); }
1041 static inline Int32x4 v_sub(Int32x4 a, Int32x4 b) { return vsubq_s32(a, b); }
1042
1043 static inline Float32x4 v_mul(Float32x4 a, Float32x4 b) { return vmulq_f32(a, b); }
1044
1045 static inline Float32x4 v_sqrt(Float32x4 x) { Float32x4 y = vrsqrteq_f32(x); y = vmulq_f32(y, vrsqrtsq_f32(x, vmulq_f32(y, y))); return vmulq_f32(x, y); }
1046
1047 static inline Int32x4 v_toInt(Float32x4 x) { return vcvtq_s32_f32(x); }
1048
1049 static inline Int32x4 v_greaterOrEqual(Float32x4 a, Float32x4 b) { return vreinterpretq_s32_u32(vcgeq_f32(a, b)); }
1050};
1051
1052const uint * QT_FASTCALL qt_fetch_radial_gradient_neon(uint *buffer, const Operator *op, const QSpanData *data,
1053 int y, int x, int length)
1054{
1055 return qt_fetch_radial_gradient_template<QRadialFetchSimd<QSimdNeon>,uint>(buffer, op, data, y, x, length);
1056}
1057
1058extern void QT_FASTCALL qt_convert_rgb888_to_rgb32_neon(quint32 *dst, const uchar *src, int len);
1059
1060const uint * QT_FASTCALL qt_fetchUntransformed_888_neon(uint *buffer, const Operator *, const QSpanData *data,
1061 int y, int x, int length)
1062{
1063 const uchar *line = data->texture.scanLine(y) + x * 3;
1064 qt_convert_rgb888_to_rgb32_neon(buffer, line, length);
1065 return buffer;
1066}
1067
1068#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
1069static inline uint32x4_t vrgba2argb(uint32x4_t srcVector)
1070{
1071#if defined(Q_PROCESSOR_ARM_64)
1072 const uint8x16_t rgbaMask = qvsetq_n_u8(2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15);
1073#else
1074 const uint8x8_t rgbaMask = qvset_n_u8(2, 1, 0, 3, 6, 5, 4, 7);
1075#endif
1076#if defined(Q_PROCESSOR_ARM_64)
1077 srcVector = vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(srcVector), rgbaMask));
1078#else
1079 // no vqtbl1q_u8, so use two vtbl1_u8
1080 const uint8x8_t low = vtbl1_u8(vreinterpret_u8_u32(vget_low_u32(srcVector)), rgbaMask);
1081 const uint8x8_t high = vtbl1_u8(vreinterpret_u8_u32(vget_high_u32(srcVector)), rgbaMask);
1082 srcVector = vcombine_u32(vreinterpret_u32_u8(low), vreinterpret_u32_u8(high));
1083#endif
1084 return srcVector;
1085}
1086
1087template<bool RGBA>
1088static inline void convertARGBToARGB32PM_neon(uint *buffer, const uint *src, int count)
1089{
1090 int i = 0;
1091 const uint8x8_t shuffleMask = qvset_n_u8(3, 3, 3, 3, 7, 7, 7, 7);
1092 const uint32x4_t blendMask = vdupq_n_u32(0xff000000);
1093
1094 for (; i < count - 3; i += 4) {
1095 uint32x4_t srcVector = vld1q_u32(src + i);
1096 uint32x4_t alphaVector = vshrq_n_u32(srcVector, 24);
1097#if defined(Q_PROCESSOR_ARM_64)
1098 uint32_t alphaSum = vaddvq_u32(alphaVector);
1099#else
1100 // no vaddvq_u32
1101 uint32x2_t tmp = vpadd_u32(vget_low_u32(alphaVector), vget_high_u32(alphaVector));
1102 uint32_t alphaSum = vget_lane_u32(vpadd_u32(tmp, tmp), 0);
1103#endif
1104 if (alphaSum) {
1105 if (alphaSum != 255 * 4) {
1106 if (RGBA)
1107 srcVector = vrgba2argb(srcVector);
1108 const uint8x8_t s1 = vreinterpret_u8_u32(vget_low_u32(srcVector));
1109 const uint8x8_t s2 = vreinterpret_u8_u32(vget_high_u32(srcVector));
1110 const uint8x8_t alpha1 = vtbl1_u8(s1, shuffleMask);
1111 const uint8x8_t alpha2 = vtbl1_u8(s2, shuffleMask);
1112 uint16x8_t src1 = vmull_u8(s1, alpha1);
1113 uint16x8_t src2 = vmull_u8(s2, alpha2);
1114 src1 = vsraq_n_u16(src1, src1, 8);
1115 src2 = vsraq_n_u16(src2, src2, 8);
1116 const uint8x8_t d1 = vrshrn_n_u16(src1, 8);
1117 const uint8x8_t d2 = vrshrn_n_u16(src2, 8);
1118 const uint32x4_t d = vbslq_u32(blendMask, srcVector, vreinterpretq_u32_u8(vcombine_u8(d1, d2)));
1119 vst1q_u32(buffer + i, d);
1120 } else {
1121 if (RGBA)
1122 vst1q_u32(buffer + i, vrgba2argb(srcVector));
1123 else if (buffer != src)
1124 vst1q_u32(buffer + i, srcVector);
1125 }
1126 } else {
1127 vst1q_u32(buffer + i, vdupq_n_u32(0));
1128 }
1129 }
1130
1131 SIMD_EPILOGUE(i, count, 3) {
1132 uint v = qPremultiply(src[i]);
1133 buffer[i] = RGBA ? RGBA2ARGB(v) : v;
1134 }
1135}
1136
1137template<bool RGBA>
1138static inline void convertARGB32ToRGBA64PM_neon(QRgba64 *buffer, const uint *src, int count)
1139{
1140 if (count <= 0)
1141 return;
1142
1143 const uint8x8_t shuffleMask = qvset_n_u8(3, 3, 3, 3, 7, 7, 7, 7);
1144 const uint64x2_t blendMask = vdupq_n_u64(Q_UINT64_C(0xffff000000000000));
1145
1146 int i = 0;
1147 for (; i < count-3; i += 4) {
1148 uint32x4_t vs32 = vld1q_u32(src + i);
1149 uint32x4_t alphaVector = vshrq_n_u32(vs32, 24);
1150#if defined(Q_PROCESSOR_ARM_64)
1151 uint32_t alphaSum = vaddvq_u32(alphaVector);
1152#else
1153 // no vaddvq_u32
1154 uint32x2_t tmp = vpadd_u32(vget_low_u32(alphaVector), vget_high_u32(alphaVector));
1155 uint32_t alphaSum = vget_lane_u32(vpadd_u32(tmp, tmp), 0);
1156#endif
1157 if (alphaSum) {
1158 if (!RGBA)
1159 vs32 = vrgba2argb(vs32);
1160 const uint8x16_t vs8 = vreinterpretq_u8_u32(vs32);
1161 const uint8x16x2_t v = vzipq_u8(vs8, vs8);
1162 if (alphaSum != 255 * 4) {
1163 const uint8x8_t s1 = vreinterpret_u8_u32(vget_low_u32(vs32));
1164 const uint8x8_t s2 = vreinterpret_u8_u32(vget_high_u32(vs32));
1165 const uint8x8_t alpha1 = vtbl1_u8(s1, shuffleMask);
1166 const uint8x8_t alpha2 = vtbl1_u8(s2, shuffleMask);
1167 uint16x8_t src1 = vmull_u8(s1, alpha1);
1168 uint16x8_t src2 = vmull_u8(s2, alpha2);
1169 // convert from 0->(255x255) to 0->(255x257)
1170 src1 = vsraq_n_u16(src1, src1, 7);
1171 src2 = vsraq_n_u16(src2, src2, 7);
1172
1173 // now restore alpha from the trivial conversion
1174 const uint64x2_t d1 = vbslq_u64(blendMask, vreinterpretq_u64_u8(v.val[0]), vreinterpretq_u64_u16(src1));
1175 const uint64x2_t d2 = vbslq_u64(blendMask, vreinterpretq_u64_u8(v.val[1]), vreinterpretq_u64_u16(src2));
1176
1177 vst1q_u16((uint16_t *)buffer, vreinterpretq_u16_u64(d1));
1178 buffer += 2;
1179 vst1q_u16((uint16_t *)buffer, vreinterpretq_u16_u64(d2));
1180 buffer += 2;
1181 } else {
1182 vst1q_u16((uint16_t *)buffer, vreinterpretq_u16_u8(v.val[0]));
1183 buffer += 2;
1184 vst1q_u16((uint16_t *)buffer, vreinterpretq_u16_u8(v.val[1]));
1185 buffer += 2;
1186 }
1187 } else {
1188 vst1q_u16((uint16_t *)buffer, vdupq_n_u16(0));
1189 buffer += 2;
1190 vst1q_u16((uint16_t *)buffer, vdupq_n_u16(0));
1191 buffer += 2;
1192 }
1193 }
1194
1195 SIMD_EPILOGUE(i, count, 3) {
1196 uint s = src[i];
1197 if (RGBA)
1198 s = RGBA2ARGB(s);
1199 *buffer++ = QRgba64::fromArgb32(s).premultiplied();
1200 }
1201}
1202
1203static inline float32x4_t reciprocal_mul_ps(float32x4_t a, float mul)
1204{
1205 float32x4_t ia = vrecpeq_f32(a); // estimate 1/a
1206 ia = vmulq_f32(vrecpsq_f32(a, ia), vmulq_n_f32(ia, mul)); // estimate improvement step * mul
1207 return ia;
1208}
1209
1210template<bool RGBA, bool RGBx>
1211static inline void convertARGBFromARGB32PM_neon(uint *buffer, const uint *src, int count)
1212{
1213 int i = 0;
1214 const uint32x4_t alphaMask = vdupq_n_u32(0xff000000);
1215
1216 for (; i < count - 3; i += 4) {
1217 uint32x4_t srcVector = vld1q_u32(src + i);
1218 uint32x4_t alphaVector = vshrq_n_u32(srcVector, 24);
1219#if defined(Q_PROCESSOR_ARM_64)
1220 uint32_t alphaSum = vaddvq_u32(alphaVector);
1221#else
1222 // no vaddvq_u32
1223 uint32x2_t tmp = vpadd_u32(vget_low_u32(alphaVector), vget_high_u32(alphaVector));
1224 uint32_t alphaSum = vget_lane_u32(vpadd_u32(tmp, tmp), 0);
1225#endif
1226 if (alphaSum) {
1227 if (alphaSum != 255 * 4) {
1228 if (RGBA)
1229 srcVector = vrgba2argb(srcVector);
1230 const float32x4_t a = vcvtq_f32_u32(alphaVector);
1231 const float32x4_t ia = reciprocal_mul_ps(a, 255.0f);
1232 // Convert 4x(4xU8) to 4x(4xF32)
1233 uint16x8_t tmp1 = vmovl_u8(vget_low_u8(vreinterpretq_u8_u32(srcVector)));
1234 uint16x8_t tmp3 = vmovl_u8(vget_high_u8(vreinterpretq_u8_u32(srcVector)));
1235 float32x4_t src1 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(tmp1)));
1236 float32x4_t src2 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(tmp1)));
1237 float32x4_t src3 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(tmp3)));
1238 float32x4_t src4 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(tmp3)));
1239 src1 = vmulq_lane_f32(src1, vget_low_f32(ia), 0);
1240 src2 = vmulq_lane_f32(src2, vget_low_f32(ia), 1);
1241 src3 = vmulq_lane_f32(src3, vget_high_f32(ia), 0);
1242 src4 = vmulq_lane_f32(src4, vget_high_f32(ia), 1);
1243 // Convert 4x(4xF32) back to 4x(4xU8) (over a 8.1 fixed point format to get rounding)
1244 tmp1 = vcombine_u16(vrshrn_n_u32(vcvtq_n_u32_f32(src1, 1), 1),
1245 vrshrn_n_u32(vcvtq_n_u32_f32(src2, 1), 1));
1246 tmp3 = vcombine_u16(vrshrn_n_u32(vcvtq_n_u32_f32(src3, 1), 1),
1247 vrshrn_n_u32(vcvtq_n_u32_f32(src4, 1), 1));
1248 uint32x4_t dstVector = vreinterpretq_u32_u8(vcombine_u8(vmovn_u16(tmp1), vmovn_u16(tmp3)));
1249 // Overwrite any undefined results from alpha==0 with zeros:
1250#if defined(Q_PROCESSOR_ARM_64)
1251 uint32x4_t srcVectorAlphaMask = vceqzq_u32(alphaVector);
1252#else
1253 uint32x4_t srcVectorAlphaMask = vceqq_u32(alphaVector, vdupq_n_u32(0));
1254#endif
1255 dstVector = vbicq_u32(dstVector, srcVectorAlphaMask);
1256 // Restore or mask alpha values:
1257 if (RGBx)
1258 dstVector = vorrq_u32(alphaMask, dstVector);
1259 else
1260 dstVector = vbslq_u32(alphaMask, srcVector, dstVector);
1261 vst1q_u32(&buffer[i], dstVector);
1262 } else {
1263 // 4xAlpha==255, no change except if we are doing RGBA->ARGB:
1264 if (RGBA)
1265 vst1q_u32(&buffer[i], vrgba2argb(srcVector));
1266 else if (buffer != src)
1267 vst1q_u32(&buffer[i], srcVector);
1268 }
1269 } else {
1270 // 4xAlpha==0, always zero, except if output is RGBx:
1271 if (RGBx)
1272 vst1q_u32(&buffer[i], alphaMask);
1273 else
1274 vst1q_u32(&buffer[i], vdupq_n_u32(0));
1275 }
1276 }
1277
1278 SIMD_EPILOGUE(i, count, 3) {
1279 uint v = qUnpremultiply(src[i]);
1280 if (RGBx)
1281 v = 0xff000000 | v;
1282 if (RGBA)
1283 v = ARGB2RGBA(v);
1284 buffer[i] = v;
1285 }
1286}
1287
1288void QT_FASTCALL convertARGB32ToARGB32PM_neon(uint *buffer, int count, const QList<QRgb> *)
1289{
1290 convertARGBToARGB32PM_neon<false>(buffer, buffer, count);
1291}
1292
1293void QT_FASTCALL convertRGBA8888ToARGB32PM_neon(uint *buffer, int count, const QList<QRgb> *)
1294{
1295 convertARGBToARGB32PM_neon<true>(buffer, buffer, count);
1296}
1297
1298const uint *QT_FASTCALL fetchARGB32ToARGB32PM_neon(uint *buffer, const uchar *src, int index, int count,
1299 const QList<QRgb> *, QDitherInfo *)
1300{
1301 convertARGBToARGB32PM_neon<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1302 return buffer;
1303}
1304
1305const uint *QT_FASTCALL fetchRGBA8888ToARGB32PM_neon(uint *buffer, const uchar *src, int index, int count,
1306 const QList<QRgb> *, QDitherInfo *)
1307{
1308 convertARGBToARGB32PM_neon<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1309 return buffer;
1310}
1311
1312const QRgba64 * QT_FASTCALL convertARGB32ToRGBA64PM_neon(QRgba64 *buffer, const uint *src, int count,
1313 const QList<QRgb> *, QDitherInfo *)
1314{
1315 convertARGB32ToRGBA64PM_neon<false>(buffer, src, count);
1316 return buffer;
1317}
1318
1319const QRgba64 * QT_FASTCALL convertRGBA8888ToRGBA64PM_neon(QRgba64 *buffer, const uint *src, int count,
1320 const QList<QRgb> *, QDitherInfo *)
1321{
1322 convertARGB32ToRGBA64PM_neon<true>(buffer, src, count);
1323 return buffer;
1324}
1325
1326const QRgba64 *QT_FASTCALL fetchARGB32ToRGBA64PM_neon(QRgba64 *buffer, const uchar *src, int index, int count,
1327 const QList<QRgb> *, QDitherInfo *)
1328{
1329 convertARGB32ToRGBA64PM_neon<false>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1330 return buffer;
1331}
1332
1333const QRgba64 *QT_FASTCALL fetchRGBA8888ToRGBA64PM_neon(QRgba64 *buffer, const uchar *src, int index, int count,
1334 const QList<QRgb> *, QDitherInfo *)
1335{
1336 convertARGB32ToRGBA64PM_neon<true>(buffer, reinterpret_cast<const uint *>(src) + index, count);
1337 return buffer;
1338}
1339
1340void QT_FASTCALL storeRGB32FromARGB32PM_neon(uchar *dest, const uint *src, int index, int count,
1341 const QList<QRgb> *, QDitherInfo *)
1342{
1343 uint *d = reinterpret_cast<uint *>(dest) + index;
1344 convertARGBFromARGB32PM_neon<false,true>(d, src, count);
1345}
1346
1347void QT_FASTCALL storeARGB32FromARGB32PM_neon(uchar *dest, const uint *src, int index, int count,
1348 const QList<QRgb> *, QDitherInfo *)
1349{
1350 uint *d = reinterpret_cast<uint *>(dest) + index;
1351 convertARGBFromARGB32PM_neon<false,false>(d, src, count);
1352}
1353
1354void QT_FASTCALL storeRGBA8888FromARGB32PM_neon(uchar *dest, const uint *src, int index, int count,
1355 const QList<QRgb> *, QDitherInfo *)
1356{
1357 uint *d = reinterpret_cast<uint *>(dest) + index;
1358 convertARGBFromARGB32PM_neon<true,false>(d, src, count);
1359}
1360
1361void QT_FASTCALL storeRGBXFromARGB32PM_neon(uchar *dest, const uint *src, int index, int count,
1362 const QList<QRgb> *, QDitherInfo *)
1363{
1364 uint *d = reinterpret_cast<uint *>(dest) + index;
1365 convertARGBFromARGB32PM_neon<true,true>(d, src, count);
1366}
1367
1368#endif // Q_BYTE_ORDER == Q_LITTLE_ENDIAN
1369
1370QT_END_NAMESPACE
1371
1372#endif // __ARM_NEON__