2 * Copyright 2014 The Android Open Source Project
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
8 #ifdef CRBUG_399842_FIXED
10 #if defined(__clang__) || (defined(__GNUC__) && !defined(SK_BUILD_FOR_MAC))
12 #define CFI_PUSH(REG) \
13 .cfi_adjust_cfa_offset 4; \
14 .cfi_rel_offset REG, 0
16 #define CFI_POP(REG) \
17 .cfi_adjust_cfa_offset -4; \
20 #define PUSH(REG) pushl REG; CFI_PUSH (REG)
21 #define POP(REG) popl REG; CFI_POP (REG)
22 #define RETURN POP(%edi); ret
24 #define EXTRACT_ALPHA(var1, var2) \
25 movdqa %var1, %var2; /* Clone source pixels to extract alpha */\
26 psrlw $8, %var2; /* Discard red and blue, leaving alpha and green */\
27 pshufhw $0xF5, %var2, %var2; /* Repeat alpha for scaling (high) */\
28 movdqa %xmm6, %xmm4; \
29 pshuflw $0xF5, %var2, %var2; /* Repeat alpha for scaling (low) */\
30 movdqa %xmm5, %xmm3; \
31 psubw %var2, %xmm4 /* Finalize alpha calculations */
33 #define SCALE_PIXELS \
34 psllw $8, %xmm5; /* Filter out red and blue components */\
35 pmulhuw %xmm4, %xmm5; /* Scale red and blue */\
36 psrlw $8, %xmm3; /* Filter out alpha and green components */\
37 pmullw %xmm4, %xmm3 /* Scale alpha and green */
41 * void S32A_Opaque_BlitRow32_SSE4(SkPMColor* SK_RESTRICT dst,
42 * const SkPMColor* SK_RESTRICT src,
43 * int count, U8CPU alpha)
45 * This function is divided into six blocks: initialization, blit 4-15 pixels,
46 * blit 0-3 pixels, align destination for 16+ pixel blits,
47 * blit 16+ pixels with source unaligned, blit 16+ pixels with source aligned.
48 * There are some code reuse between the blocks.
50 * The primary optimization comes from checking the source pixels' alpha value.
51 * If the alpha is zero, the pixel can be skipped entirely.
52 * If the alpha is fully opaque, the pixel can be copied directly to the destination.
53 * According to collected statistics, these two cases are the most common.
54 * The main loop(s) uses pre-loading and unrolling in an attempt to reduce the
55 * memory latency worse-case.
61 .section .text.sse4.2,"ax",@progbits
62 .type S32A_Opaque_BlitRow32_SSE4_asm, @function
65 #if defined(SK_BUILD_FOR_MAC)
66 .global _S32A_Opaque_BlitRow32_SSE4_asm
67 .private_extern _S32A_Opaque_BlitRow32_SSE4_asm
68 _S32A_Opaque_BlitRow32_SSE4_asm:
70 .global S32A_Opaque_BlitRow32_SSE4_asm
71 .hidden S32A_Opaque_BlitRow32_SSE4_asm
72 S32A_Opaque_BlitRow32_SSE4_asm:
75 movl 8(%esp), %eax // Source pointer
76 movl 12(%esp), %ecx // Pixel count
77 movl 4(%esp), %edx // Destination pointer
80 // Setup SSE constants
81 pcmpeqd %xmm7, %xmm7 // 0xFF000000 mask to check alpha
83 pcmpeqw %xmm6, %xmm6 // 16-bit 256 to calculate inv. alpha
86 pcmpeqw %xmm0, %xmm0 // 0x00FF00FF mask (Must be in xmm0 because of pblendvb)
88 subl $4, %ecx // Check if we have only 0-3 pixels
91 cmpl $11, %ecx // Do we have enough pixels to run the main loop?
94 // Handle small blits (4-15 pixels)
95 ////////////////////////////////////////////////////////////////////////////////
96 xorl %edi, %edi // Reset offset to zero
99 lddqu (%eax, %edi), %xmm1 // Load four source pixels
100 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
101 ja .LSmallAlphaNotOpaqueOrZero
102 jz .LSmallAlphaZero // If all alphas are zero, skip the pixels completely
103 movdqu %xmm1, (%edx, %edi) // Store four destination pixels
106 subl $4, %ecx // Check if there are four additional pixels, at least
110 // Handle mixed alphas (calculate and scale)
112 .LSmallAlphaNotOpaqueOrZero:
113 lddqu (%edx, %edi), %xmm5 // Load four destination pixels
114 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
115 SCALE_PIXELS // Scale pixels using alpha
118 subl $4, %ecx // Check if there are four additional pixels, at least
119 pblendvb %xmm5, %xmm3 // Mask in %xmm0, implicitly
120 paddb %xmm3, %xmm1 // Add source and destination pixels together
121 movdqu %xmm1, -16(%edx, %edi) // Store four destination pixels
124 // Handle the last 0-3 pixels (also used by the main loops)
126 cmpl $-4, %ecx // Check if we are done
128 sall $2, %ecx // Calculate offset for last pixels
131 lddqu (%eax, %edi), %xmm1 // Load last four source pixels (overlapping)
132 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
133 jc .LSmallRemainingStoreAll// If all alphas are opaque, just store (overlapping)
134 jz .LSmallExit // If all alphas are zero, skip the pixels completely
136 // Handle mixed alphas (calculate and scale)
137 lddqu (%edx, %edi), %xmm5 // Load last four destination pixels (overlapping)
138 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
140 psllw $8, %xmm3 // Filter out red and blue components
141 pmulhuw %xmm4, %xmm3 // Scale red and blue
143 psrlw $8, %xmm2 // Filter out alpha and green components
144 pmullw %xmm4, %xmm2 // Scale alpha and green
146 cmpl $-8, %ecx // Check how many pixels should be written
147 pblendvb %xmm3, %xmm2 // Combine results (mask in %xmm0, implicitly)
148 paddb %xmm2, %xmm1 // Add source and destination pixels together
149 jb .LSmallPixelsLeft1
150 ja .LSmallPixelsLeft3 // To avoid double-blending the overlapping pixels...
151 pblendw $0xF0, %xmm1, %xmm5 // Merge only the final two pixels to the destination
152 movdqu %xmm5, (%edx, %edi) // Store last two destination pixels
157 pblendw $0xC0, %xmm1, %xmm5 // Merge only the final pixel to the destination
158 movdqu %xmm5, (%edx, %edi) // Store last destination pixel
162 pblendw $0xFC, %xmm1, %xmm5 // Merge only the final three pixels to the destination
163 movdqu %xmm5, (%edx, %edi) // Store last three destination pixels
166 .LSmallRemainingStoreAll:
167 movdqu %xmm1, (%edx, %edi) // Store last destination pixels (overwrite)
170 // Handle really small blits (0-3 pixels)
171 ////////////////////////////////////////////////////////////////////////////////
174 jle .LReallySmallExit
176 cmp $2, %ecx // Check how many pixels should be read
177 pinsrd $0x0, (%eax), %xmm1 // Load one source pixel
178 pinsrd $0x0, (%edx), %xmm5 // Load one destination pixel
180 pinsrd $0x1, 4(%eax), %xmm1 // Load second source pixel
181 pinsrd $0x1, 4(%edx), %xmm5 // Load second destination pixel
183 pinsrd $0x2, 8(%eax), %xmm1 // Load third source pixel
184 pinsrd $0x2, 8(%edx), %xmm5 // Load third destination pixel
187 ptest %xmm7, %xmm1 // Check if all alphas are opaque
188 jc .LReallySmallStore // If all alphas are opaque, just store
190 // Handle mixed alphas (calculate and scale)
191 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
193 pand %xmm0, %xmm5 // Filter out red and blue components
194 pmullw %xmm4, %xmm5 // Scale red and blue
195 psrlw $8, %xmm3 // Filter out alpha and green components
196 pmullw %xmm4, %xmm3 // Scale alpha and green
198 psrlw $8, %xmm5 // Combine results
199 pblendvb %xmm5, %xmm3 // Mask in %xmm0, implicitly
200 paddb %xmm3, %xmm1 // Add source and destination pixels together
203 cmp $2, %ecx // Check how many pixels should be written
204 pextrd $0x0, %xmm1, (%edx) // Store one destination pixel
206 pextrd $0x1, %xmm1, 4(%edx) // Store second destination pixel
208 pextrd $0x2, %xmm1, 8(%edx) // Store third destination pixel
212 // Handle bigger blit operations (16+ pixels)
213 ////////////////////////////////////////////////////////////////////////////////
216 // Align destination?
218 lddqu (%eax), %xmm1 // Pre-load four source pixels
221 movl %edx, %edi // Calculate alignment of destination pointer
225 // Handle 1-3 pixels to align destination
226 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
227 jz .LAlignDone // If all alphas are zero, just skip
228 lddqu (%edx), %xmm5 // Load four destination pixels
229 jc .LAlignStore // If all alphas are opaque, just store
231 // Handle mixed alphas (calculate and scale)
232 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
234 psllw $8, %xmm3 // Filter out red and blue components
235 pmulhuw %xmm4, %xmm3 // Scale red and blue
237 psrlw $8, %xmm2 // Filter out alpha and green components
238 pmullw %xmm4, %xmm2 // Scale alpha and green
240 pblendvb %xmm3, %xmm2 // Combine results (mask in %xmm0, implicitly)
241 paddb %xmm2, %xmm1 // Add source and destination pixels together
244 cmp $8, %edi // Check how many pixels should be written
245 jb .LAlignPixelsLeft1
246 ja .LAlignPixelsLeft3
247 pblendw $0x0F, %xmm1, %xmm5 // Blend two pixels
248 jmp .LAlignStorePixels
251 pblendw $0x03, %xmm1, %xmm5 // Blend one pixel
252 jmp .LAlignStorePixels
255 pblendw $0x3F, %xmm1, %xmm5 // Blend three pixels
258 movdqu %xmm5, (%edx) // Store destination pixels
261 addl %edi, %eax // Adjust pointers and pixel count
264 lddqu (%eax), %xmm1 // Pre-load new source pixels (after alignment)
267 .LAligned: // Destination is guaranteed to be 16 byte aligned
268 xorl %edi, %edi // Reset offset to zero
269 subl $8, %ecx // Decrease counter (Reserve four pixels for the cleanup)
270 testl $0xF, %eax // Check alignment of source pointer
273 // Source not aligned to destination
274 ////////////////////////////////////////////////////////////////////////////////
276 .LUnalignedLoop: // Main loop for unaligned, handles eight pixels per iteration
277 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
278 ja .LAlphaNotOpaqueOrZero00
279 lddqu 16(%eax, %edi), %xmm2 // Pre-load four source pixels
281 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
284 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
285 ja .LAlphaNotOpaqueOrZero01
286 lddqu 32(%eax, %edi), %xmm1 // Pre-load four source pixels
288 movdqa %xmm2, 16(%edx, %edi) // Store four destination pixels
291 addl $32, %edi // Adjust offset and pixel count
294 addl $8, %ecx // Adjust pixel count
298 .LAlphaNotOpaqueOrZero00:
299 movdqa (%edx, %edi), %xmm5 // Load four destination pixels
300 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
301 SCALE_PIXELS // Scale pixels using alpha
303 lddqu 16(%eax, %edi), %xmm2 // Pre-load four source pixels
304 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
305 paddb %xmm3, %xmm1 // Add source and destination pixels together
306 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
308 // Handle next four pixels
309 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
310 ja .LAlphaNotOpaqueOrZero01
311 lddqu 32(%eax, %edi), %xmm1 // Pre-load four source pixels
313 movdqa %xmm2, 16(%edx, %edi) // Store four destination pixels
315 addl $32, %edi // Adjust offset and pixel count
318 addl $8, %ecx // Adjust pixel count
322 .LAlphaNotOpaqueOrZero01:
323 movdqa 16(%edx, %edi), %xmm5 // Load four destination pixels
324 EXTRACT_ALPHA(xmm2, xmm1) // Extract and clone alpha value
325 SCALE_PIXELS // Scale pixels using alpha
327 lddqu 32(%eax, %edi), %xmm1 // Pre-load four source pixels
329 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
330 paddb %xmm3, %xmm2 // Add source and destination pixels together
332 movdqa %xmm2, -16(%edx, %edi) // Store four destination pixels
334 addl $8, %ecx // Adjust pixel count
336 // Cleanup - handle pending pixels from loop
338 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
339 ja .LAlphaNotOpaqueOrZero02
341 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
345 js .LSmallRemaining // Reuse code from small loop
348 lddqu (%eax, %edi), %xmm1 // Load four source pixels
349 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
350 ja .LAlphaNotOpaqueOrZero02
352 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
356 jmp .LSmallRemaining // Reuse code from small loop
358 .LAlphaNotOpaqueOrZero02:
359 movdqa (%edx, %edi), %xmm5 // Load four destination pixels
360 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
361 SCALE_PIXELS // Scale pixels using alpha
365 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
366 paddb %xmm3, %xmm1 // Add source and destination pixels together
367 movdqa %xmm1, -16(%edx, %edi) // Store four destination pixels
368 js .LSmallRemaining // Reuse code from small loop
371 // Source aligned to destination
372 ////////////////////////////////////////////////////////////////////////////////
374 .LAlignedLoop: // Main loop for aligned, handles eight pixels per iteration
375 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
376 ja .LAlphaNotOpaqueOrZero10
377 movdqa 16(%eax, %edi), %xmm2 // Pre-load four source pixels
379 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
382 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
383 ja .LAlphaNotOpaqueOrZero11
384 movdqa 32(%eax, %edi), %xmm1 // Pre-load four source pixels
386 movdqa %xmm2, 16(%edx, %edi) // Store four destination pixels
389 addl $32, %edi // Adjust offset and pixel count
392 addl $8, %ecx // Adjust pixel count
396 .LAlphaNotOpaqueOrZero10:
397 movdqa (%edx, %edi), %xmm5 // Load four destination pixels
398 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
399 SCALE_PIXELS // Scale pixels using alpha
401 movdqa 16(%eax, %edi), %xmm2 // Pre-load four source pixels
402 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
403 paddb %xmm3, %xmm1 // Add source and destination pixels together
404 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
406 // Handle next four pixels
407 ptest %xmm7, %xmm2 // Check if all alphas are zero or opaque
408 ja .LAlphaNotOpaqueOrZero11
409 movdqa 32(%eax, %edi), %xmm1 // Pre-load four source pixels
411 movdqa %xmm2, 16(%edx, %edi) // Store four destination pixels
413 addl $32, %edi // Adjust offset and pixel count
416 addl $8, %ecx // Adjust pixel count
420 .LAlphaNotOpaqueOrZero11:
421 movdqa 16(%edx, %edi), %xmm5 // Load four destination pixels
422 EXTRACT_ALPHA(xmm2, xmm1) // Extract and clone alpha value
423 SCALE_PIXELS // Scale pixels using alpha
424 movdqa 32(%eax, %edi), %xmm1 // Pre-load four source pixels
427 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
428 paddb %xmm3, %xmm2 // Add source and destination pixels together
430 movdqa %xmm2, -16(%edx, %edi) // Store four destination pixels
432 addl $8, %ecx // Adjust pixel count
434 // Cleanup - handle pending pixels from loop
436 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
437 ja .LAlphaNotOpaqueOrZero12
439 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
443 js .LSmallRemaining // Reuse code from small loop
446 movdqa (%eax, %edi), %xmm1 // Load four source pixels
447 ptest %xmm7, %xmm1 // Check if all alphas are zero or opaque
448 ja .LAlphaNotOpaqueOrZero12
450 movdqa %xmm1, (%edx, %edi) // Store four destination pixels
454 jmp .LSmallRemaining // Reuse code from small loop
456 .LAlphaNotOpaqueOrZero12:
457 movdqa (%edx, %edi), %xmm5 // Load four destination pixels
458 EXTRACT_ALPHA(xmm1, xmm2) // Extract and clone alpha value
459 SCALE_PIXELS // Scale pixels using alpha
463 pblendvb %xmm5, %xmm3 // Combine results (mask in %xmm0, implicitly)
464 paddb %xmm3, %xmm1 // Add source and destination pixels together
465 movdqa %xmm1, -16(%edx, %edi) // Store four destination pixels
466 js .LSmallRemaining // Reuse code from small loop
471 .size S32A_Opaque_BlitRow32_SSE4_asm, .-S32A_Opaque_BlitRow32_SSE4_asm
475 #endif // CRBUG_399842_FIXED