2 * Copyright 2014 The Android Open Source Project
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
8 #ifdef CRBUG_399842_FIXED
10 #if defined(__clang__) || (defined(__GNUC__) && !defined(SK_BUILD_FOR_MAC))
12 #define EXTRACT_ALPHA(var1, var2) \
13 movdqa %var1, %var2; /* Clone source pixels to extract alpha */\
14 psrlw $8, %var2; /* Discard red and blue, leaving alpha and green */\
15 pshufhw $0xF5, %var2, %var2; /* Repeat alpha for scaling (high) */\
16 movdqa %xmm6, %xmm4; \
17 pshuflw $0xF5, %var2, %var2; /* Repeat alpha for scaling (low) */\
18 movdqa %xmm5, %xmm3; \
19 psubw %var2, %xmm4 /* Finalize alpha calculations */
21 #define SCALE_PIXELS \
22 psllw $8, %xmm5; /* Filter out red and blue components */\
23 pmulhuw %xmm4, %xmm5; /* Scale red and blue */\
24 psrlw $8, %xmm3; /* Filter out alpha and green components */\
25 pmullw %xmm4, %xmm3 /* Scale alpha and green */
29 * void S32A_Opaque_BlitRow32_SSE4(SkPMColor* SK_RESTRICT dst,
30 * const SkPMColor* SK_RESTRICT src,
31 * int count, U8CPU alpha)
33 * This function is divided into six blocks: initialization, blit 4-15 pixels,
34 * blit 0-3 pixels, align destination for 16+ pixel blits,
35 * blit 16+ pixels with source unaligned, blit 16+ pixels with source aligned.
36 * There are some code reuse between the blocks.
38 * The primary optimization comes from checking the source pixels' alpha value.
39 * If the alpha is zero, the pixel can be skipped entirely.
40 * If the alpha is fully opaque, the pixel can be copied directly to the destination.
41 * According to collected statistics, these two cases are the most common.
42 * The main loop(s) uses pre-loading and unrolling in an attempt to reduce the
43 * memory latency worse-case.
49 .section .text.sse4.2,"ax",@progbits
50 .type S32A_Opaque_BlitRow32_SSE4_asm, @function
53 #if defined(SK_BUILD_FOR_MAC)
54 .global _S32A_Opaque_BlitRow32_SSE4_asm
55 .private_extern _S32A_Opaque_BlitRow32_SSE4_asm
56 _S32A_Opaque_BlitRow32_SSE4_asm:
58 .global S32A_Opaque_BlitRow32_SSE4_asm
59 .hidden S32A_Opaque_BlitRow32_SSE4_asm
60 S32A_Opaque_BlitRow32_SSE4_asm:
64 movl %edx, %ecx // Pixel count
65 movq %rdi, %rdx // Destination pointer
66 movq %rsi, %rax // Source pointer
68 // Setup SSE constants
69 movdqa .LAlphaCheckMask(%rip), %xmm7 // 0xFF000000 mask to check alpha
70 movdqa .LInverseAlphaCalc(%rip), %xmm6// 16-bit 256 to calculate inv. alpha
71 movdqa .LResultMergeMask(%rip), %xmm0 // 0x00FF00FF mask (Must be in xmm0 because of pblendvb)
73 subl $4, %ecx // Check if we have only 0-3 pixels
75 cmpl $11, %ecx // Do we have enough pixels to run the main loop?
78 // Handle small blits (4-15 pixels)
79 ////////////////////////////////////////////////////////////////////////////////
80 xorq %rdi, %rdi // Reset offset to zero
83 lddqu (%rax, %rdi), %xmm1 // Load four source pixels
84 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
85 ja .LSmallAlphaNotOpaqueOrZero
87 movdqu %xmm1, (%rdx, %rdi) // Store four destination pixels
90 subl $4, %ecx // Check if there are four additional pixels, at least
94 // Handle mixed alphas (calculate and scale)
96 .LSmallAlphaNotOpaqueOrZero:
97 lddqu (%rdx, %rdi), %xmm5 // Load four destination pixels
98 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
99 SCALE_PIXELS // Scale pixels using alpha
102 subl $4, %ecx // Check if there are four additional pixels, at least
103 pblendvb %xmm5, %xmm3 // Mask in %xmm0, implicitly
104 paddb %xmm3, %xmm1 // Add source and destination pixels together
105 movdqu %xmm1, -16(%rdx, %rdi) // Store four destination pixels
108 // Handle the last 0-3 pixels (also used by the main loops)
110 cmpl $-4, %ecx // Check if we are done
112 sall $2, %ecx // Calculate offset for last pixels
116 lddqu (%rax, %rdi), %xmm1 // Load last four source pixels (overlapping)
117 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
118 jc .LSmallRemainingStoreAll// If all alphas are opaque, just store (overlapping)
119 jz .LSmallExit // If all alphas are zero, skip the pixels completely
121 // Handle mixed alphas (calculate and scale)
122 lddqu (%rdx, %rdi), %xmm5 // Load last four destination pixels (overlapping)
123 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
125 psllw $8, %xmm3 // Filter out red and blue components
126 pmulhuw %xmm4, %xmm3 // Scale red and blue
128 psrlw $8, %xmm2 // Filter out alpha and green components
129 pmullw %xmm4, %xmm2 // Scale alpha and green
131 cmpl $-8, %ecx // Check how many pixels should be written
132 pblendvb %xmm3, %xmm2 // Combine results (mask in %xmm0, implicitly)
133 paddb %xmm2, %xmm1 // Add source and destination pixels together
134 jb .LSmallPixelsLeft1
135 ja .LSmallPixelsLeft3 // To avoid double-blending the overlapping pixels...
136 pblendw $0xF0, %xmm1, %xmm5 // Merge only the final two pixels to the destination
137 movdqu %xmm5, (%rdx, %rdi) // Store last two destination pixels
142 pblendw $0xC0, %xmm1, %xmm5 // Merge only the final pixel to the destination
143 movdqu %xmm5, (%rdx, %rdi) // Store last destination pixel
147 pblendw $0xFC, %xmm1, %xmm5 // Merge only the final three pixels to the destination
148 movdqu %xmm5, (%rdx, %rdi) // Store last three destination pixels
151 .LSmallRemainingStoreAll:
152 movdqu %xmm1, (%rdx, %rdi) // Store last destination pixels (overwrite)
155 // Handle really small blits (0-3 pixels)
156 ////////////////////////////////////////////////////////////////////////////////
159 jle .LReallySmallExit
161 cmpl $2, %ecx // Check how many pixels should be read
162 pinsrd $0x0, (%rax), %xmm1 // Load one source pixel
163 pinsrd $0x0, (%rdx), %xmm5 // Load one destination pixel
165 pinsrd $0x1, 4(%rax), %xmm1 // Load second source pixel
166 pinsrd $0x1, 4(%rdx), %xmm5 // Load second destination pixel
168 pinsrd $0x2, 8(%rax), %xmm1 // Load third source pixel
169 pinsrd $0x2, 8(%rdx), %xmm5 // Load third destination pixel
172 ptest %xmm7, %xmm1 // Check if all alphas are opaque
173 jc .LReallySmallStore // If all alphas are opaque, just store
175 // Handle mixed alphas (calculate and scale)
176 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
178 pand %xmm0, %xmm5 // Filter out red and blue components
179 pmullw %xmm4, %xmm5 // Scale red and blue
180 psrlw $8, %xmm3 // Filter out alpha and green components
181 pmullw %xmm4, %xmm3 // Scale alpha and green
183 psrlw $8, %xmm5 // Combine results
184 pblendvb %xmm5, %xmm3 // Mask in %xmm0, implicitly
185 paddb %xmm3, %xmm1 // Add source and destination pixels together
188 cmpl $2, %ecx // Check how many pixels should be written
189 pextrd $0x0, %xmm1, (%rdx) // Store one destination pixel
191 pextrd $0x1, %xmm1, 4(%rdx) // Store second destination pixel
193 pextrd $0x2, %xmm1, 8(%rdx) // Store third destination pixel
197 // Handle bigger blit operations (16+ pixels)
198 ////////////////////////////////////////////////////////////////////////////////
201 // Align destination?
203 lddqu (%rax), %xmm1 // Pre-load four source pixels
206 movq %rdx, %rdi // Calculate alignment of destination pointer
210 // Handle 1-3 pixels to align destination
211 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
212 jz .LAlignDone // If all alphas are zero, just skip
213 lddqu (%rdx), %xmm5 // Load four destination pixels
214 jc .LAlignStore // If all alphas are opaque, just store
216 // Handle mixed alphas (calculate and scale)
217 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
219 psllw $8, %xmm3 // Filter out red and blue components
220 pmulhuw %xmm4, %xmm3 // Scale red and blue
222 psrlw $8, %xmm2 // Filter out alpha and green components
223 pmullw %xmm4, %xmm2 // Scale alpha and green
225 pblendvb %xmm3, %xmm2 // Combine results (mask in %xmm0, implicitly)
226 paddb %xmm2, %xmm1 // Add source and destination pixels together
229 cmpl $8, %edi // Check how many pixels should be written
230 jb .LAlignPixelsLeft1
231 ja .LAlignPixelsLeft3
232 pblendw $0x0F, %xmm1, %xmm5 // Blend two pixels
233 jmp .LAlignStorePixels
236 pblendw $0x03, %xmm1, %xmm5 // Blend one pixel
237 jmp .LAlignStorePixels
240 pblendw $0x3F, %xmm1, %xmm5 // Blend three pixels
243 movdqu %xmm5, (%rdx) // Store destination pixels
246 addq %rdi, %rax // Adjust pointers and pixel count
249 lddqu (%rax), %xmm1 // Pre-load new source pixels (after alignment)
252 .LAligned: // Destination is guaranteed to be 16 byte aligned
253 xorq %rdi, %rdi // Reset offset to zero
254 subl $8, %ecx // Decrease counter (Reserve four pixels for the cleanup)
255 testl $0xF, %eax // Check alignment of source pointer
258 // Source not aligned to destination
259 ////////////////////////////////////////////////////////////////////////////////
261 .LUnalignedLoop: // Main loop for unaligned, handles eight pixels per iteration
262 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
263 ja .LAlphaNotOpaqueOrZero00
264 lddqu 16(%rax, %rdi), %xmm2 // Pre-load four source pixels
266 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
269 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
270 ja .LAlphaNotOpaqueOrZero01
271 lddqu 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
273 movdqa %xmm2, 16(%rdx, %rdi) // Store four destination pixels
276 addq $32, %rdi // Adjust offset and pixel count
279 addl $8, %ecx // Adjust pixel count
283 .LAlphaNotOpaqueOrZero00:
284 movdqa (%rdx, %rdi), %xmm5 // Load four destination pixels
285 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
286 SCALE_PIXELS // Scale pixels using alpha
288 lddqu 16(%rax, %rdi), %xmm2 // Pre-load four source pixels
289 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
290 paddb %xmm3, %xmm1 // Add source and destination pixels together
291 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
293 // Handle next four pixels
294 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
295 ja .LAlphaNotOpaqueOrZero01
296 lddqu 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
298 movdqa %xmm2, 16(%rdx, %rdi) // Store four destination pixels
300 addq $32, %rdi // Adjust offset and pixel count
303 addl $8, %ecx // Adjust pixel count
307 .LAlphaNotOpaqueOrZero01:
308 movdqa 16(%rdx, %rdi), %xmm5 // Load four destination pixels
309 EXTRACT_ALPHA(xmm2, xmm1) // Extract and clone alpha value
310 SCALE_PIXELS // Scale pixels using alpha
312 lddqu 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
314 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
315 paddb %xmm3, %xmm2 // Add source and destination pixels together
317 movdqa %xmm2, -16(%rdx, %rdi) // Store four destination pixels
319 addl $8, %ecx // Adjust pixel count
321 // Cleanup - handle pending pixels from loop
323 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
324 ja .LAlphaNotOpaqueOrZero02
326 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
330 js .LSmallRemaining // Reuse code from small loop
333 lddqu (%rax, %rdi), %xmm1 // Load four source pixels
334 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
335 ja .LAlphaNotOpaqueOrZero02
337 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
341 jmp .LSmallRemaining // Reuse code from small loop
343 .LAlphaNotOpaqueOrZero02:
344 movdqa (%rdx, %rdi), %xmm5 // Load four destination pixels
345 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
346 SCALE_PIXELS // Scale pixels using alpha
350 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
351 paddb %xmm3, %xmm1 // Add source and destination pixels together
352 movdqa %xmm1, -16(%rdx, %rdi) // Store four destination pixels
353 js .LSmallRemaining // Reuse code from small loop
356 // Source aligned to destination
357 ////////////////////////////////////////////////////////////////////////////////
359 .LAlignedLoop: // Main loop for aligned, handles eight pixels per iteration
360 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
361 ja .LAlphaNotOpaqueOrZero10
362 movdqa 16(%rax, %rdi), %xmm2 // Pre-load four source pixels
364 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
367 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
368 ja .LAlphaNotOpaqueOrZero11
369 movdqa 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
371 movdqa %xmm2, 16(%rdx, %rdi) // Store four destination pixels
374 addq $32, %rdi // Adjust offset and pixel count
377 addl $8, %ecx // Adjust pixel count
381 .LAlphaNotOpaqueOrZero10:
382 movdqa (%rdx, %rdi), %xmm5 // Load four destination pixels
383 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
384 SCALE_PIXELS // Scale pixels using alpha
386 movdqa 16(%rax, %rdi), %xmm2 // Pre-load four source pixels
387 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
388 paddb %xmm3, %xmm1 // Add source and destination pixels together
389 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
391 // Handle next four pixels
392 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
393 ja .LAlphaNotOpaqueOrZero11
394 movdqa 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
396 movdqa %xmm2, 16(%rdx, %rdi) // Store four destination pixels
398 addq $32, %rdi // Adjust offset and pixel count
401 addl $8, %ecx // Adjust pixel count
405 .LAlphaNotOpaqueOrZero11:
406 movdqa 16(%rdx, %rdi), %xmm5 // Load four destination pixels
407 EXTRACT_ALPHA(xmm2, xmm1) // Extract and clone alpha value
408 SCALE_PIXELS // Scale pixels using alpha
409 movdqa 32(%rax, %rdi), %xmm1 // Pre-load four source pixels
412 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
413 paddb %xmm3, %xmm2 // Add source and destination pixels together
415 movdqa %xmm2, -16(%rdx, %rdi) // Store four destination pixels
417 addl $8, %ecx // Adjust pixel count
419 // Cleanup - handle four pending pixels from loop
421 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
422 ja .LAlphaNotOpaqueOrZero12
424 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
428 js .LSmallRemaining // Reuse code from small loop
431 movdqa (%rax, %rdi), %xmm1 // Pre-load four source pixels
432 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
433 ja .LAlphaNotOpaqueOrZero12
435 movdqa %xmm1, (%rdx, %rdi) // Store four destination pixels
439 jmp .LSmallRemaining // Reuse code from small loop
441 .LAlphaNotOpaqueOrZero12:
442 movdqa (%rdx, %rdi), %xmm5 // Load four destination pixels
443 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
444 SCALE_PIXELS // Scale pixels using alpha
448 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
449 paddb %xmm3, %xmm1 // Add source and destination pixels together
450 movdqa %xmm1, -16(%rdx, %rdi) // Store four destination pixels
451 js .LSmallRemaining // Reuse code from small loop
456 .size S32A_Opaque_BlitRow32_SSE4_asm, .-S32A_Opaque_BlitRow32_SSE4_asm
459 // Constants for SSE code
465 .long 0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000
467 .word 256, 256, 256, 256, 256, 256, 256, 256
469 .long 0x00FF00FF, 0x00FF00FF, 0x00FF00FF, 0x00FF00FF
472 #endif // CRBUG_399842_FIXED