2 * Copyright © 2011 SCore Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
23 * Author: Siarhei Siamashka (siarhei.siamashka@nokia.com)
24 * Author: Taekyun Kim (tkq.kim@samsung.com)
28 * This file contains scaled bilinear scanline functions implemented
29 * using older siarhei's bilinear macro template.
31 * << General scanline function procedures >>
32 * 1. bilinear interpolate source pixels
34 * 3. load destination pixels
35 * 4. duplicate mask to fill whole register
36 * 5. interleave source & destination pixels
37 * 6. apply mask to source pixels
38 * 7. combine source & destination pixels
39 * 8, Deinterleave final result
40 * 9. store destination pixels
42 * All registers with single number (i.e. src0, tmp0) are 64-bits registers.
43 * Registers with double numbers(src01, dst01) are 128-bits registers.
44 * All temp registers can be used freely outside the code block.
45 * Assume that symbol(register .req) OUT and MASK are defined at caller of these macro blocks.
48 * Support 0565 pixel format
49 * Optimization for two and last pixel cases
52 * There can be lots of pipeline stalls inside code block and between code blocks.
53 * Further optimizations will be done by new macro templates using head/tail_head/tail scheme.
56 /* Prevent the stack from becoming executable for no reason... */
57 #if defined(__linux__) && defined (__ELF__)
58 .section .note.GNU-stack,"",%progbits
71 #include "pixman-arm-neon-asm.h"
74 * Bilinear macros from pixman-arm-neon-asm.S
77 /* Supplementary macro for setting function attributes */
78 .macro pixman_asm_function fname
83 .type fname, %function
89 * Bilinear scaling support code which tries to provide pixel fetching, color
90 * format conversion, and interpolation as separate macros which can be used
91 * as the basic building blocks for constructing bilinear scanline functions.
94 .macro bilinear_load_8888 reg1, reg2, tmp
97 add TMP1, TOP, TMP2, asl #2
98 add TMP2, BOTTOM, TMP2, asl #2
99 vld1.32 {reg1}, [TMP1]
100 vld1.32 {reg2}, [TMP2]
103 .macro bilinear_load_0565 reg1, reg2, tmp
106 add TMP1, TOP, TMP2, asl #1
107 add TMP2, BOTTOM, TMP2, asl #1
108 vld1.32 {reg2[0]}, [TMP1]
109 vld1.32 {reg2[1]}, [TMP2]
110 convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
113 .macro bilinear_load_and_vertical_interpolate_two_8888 \
114 acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2
116 bilinear_load_8888 reg1, reg2, tmp1
117 vmull.u8 acc1, reg1, d28
118 vmlal.u8 acc1, reg2, d29
119 bilinear_load_8888 reg3, reg4, tmp2
120 vmull.u8 acc2, reg3, d28
121 vmlal.u8 acc2, reg4, d29
124 .macro bilinear_load_and_vertical_interpolate_four_8888 \
125 xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
126 yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
128 bilinear_load_and_vertical_interpolate_two_8888 \
129 xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
130 bilinear_load_and_vertical_interpolate_two_8888 \
131 yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
134 .macro bilinear_load_and_vertical_interpolate_two_0565 \
135 acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
141 add TMP1, TOP, TMP2, asl #1
142 add TMP2, BOTTOM, TMP2, asl #1
143 add TMP3, TOP, TMP4, asl #1
144 add TMP4, BOTTOM, TMP4, asl #1
145 vld1.32 {acc2lo[0]}, [TMP1]
146 vld1.32 {acc2hi[0]}, [TMP3]
147 vld1.32 {acc2lo[1]}, [TMP2]
148 vld1.32 {acc2hi[1]}, [TMP4]
149 convert_0565_to_x888 acc2, reg3, reg2, reg1
154 vmull.u8 acc1, reg1, d28
155 vmlal.u8 acc1, reg2, d29
156 vmull.u8 acc2, reg3, d28
157 vmlal.u8 acc2, reg4, d29
160 .macro bilinear_load_and_vertical_interpolate_four_0565 \
161 xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
162 yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
168 add TMP1, TOP, TMP2, asl #1
169 add TMP2, BOTTOM, TMP2, asl #1
170 add TMP3, TOP, TMP4, asl #1
171 add TMP4, BOTTOM, TMP4, asl #1
172 vld1.32 {xacc2lo[0]}, [TMP1]
173 vld1.32 {xacc2hi[0]}, [TMP3]
174 vld1.32 {xacc2lo[1]}, [TMP2]
175 vld1.32 {xacc2hi[1]}, [TMP4]
176 convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
181 add TMP1, TOP, TMP2, asl #1
182 add TMP2, BOTTOM, TMP2, asl #1
183 add TMP3, TOP, TMP4, asl #1
184 add TMP4, BOTTOM, TMP4, asl #1
185 vld1.32 {yacc2lo[0]}, [TMP1]
187 vld1.32 {yacc2hi[0]}, [TMP3]
189 vld1.32 {yacc2lo[1]}, [TMP2]
191 vld1.32 {yacc2hi[1]}, [TMP4]
193 convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
194 vmull.u8 xacc1, xreg1, d28
196 vmlal.u8 xacc1, xreg2, d29
198 vmull.u8 xacc2, xreg3, d28
200 vmlal.u8 xacc2, xreg4, d29
202 vmull.u8 yacc1, yreg1, d28
203 vmlal.u8 yacc1, yreg2, d29
204 vmull.u8 yacc2, yreg3, d28
205 vmlal.u8 yacc2, yreg4, d29
208 .macro bilinear_store_8888 numpix, tmp1, tmp2
210 vst1.32 {d0, d1}, [OUT]!
214 vst1.32 {d0[0]}, [OUT, :32]!
216 .error bilinear_store_8888 numpix is unsupported
220 .macro bilinear_store_0565 numpix, tmp1, tmp2
225 convert_8888_to_0565 d2, d1, d0, q1, tmp1, tmp2
229 vst1.32 {d2[0]}, [OUT]!
231 vst1.16 {d2[0]}, [OUT]!
233 .error bilinear_store_0565 numpix is unsupported
239 * Macros for loading mask pixels into register 'mask'.
240 * vdup must be done in somewhere else.
242 .macro bilinear_load_mask_x numpix, mask
245 .macro bilinear_load_mask_8 numpix, mask
247 vld1.32 {mask[0]}, [MASK]!
249 vld1.16 {mask[0]}, [MASK]!
251 vld1.8 {mask[0]}, [MASK]!
253 .error bilinear_load_mask_8 numpix is unsupported
257 .macro bilinear_load_mask mask_fmt, numpix, mask
258 bilinear_load_mask_&mask_fmt numpix, mask
263 * Macros for loading destination pixels into register 'dst0' and 'dst1'.
264 * Interleave should be done somewhere else.
266 .macro bilinear_load_dst_0565_src numpix, dst0, dst1, dst01
269 .macro bilinear_load_dst_8888_src numpix, dst0, dst1, dst01
272 .macro bilinear_load_dst_8888 numpix, dst0, dst1, dst01
274 vld1.32 {dst0, dst1}, [OUT]
276 vld1.32 {dst0}, [OUT]
278 vld1.32 {dst0[0]}, [OUT]
280 .error bilinear_load_dst_8888 numpix is unsupported
284 .macro bilinear_load_dst_8888_over numpix, dst0, dst1, dst01
285 bilinear_load_dst_8888 numpix, dst0, dst1, dst01
288 .macro bilinear_load_dst_8888_add numpix, dst0, dst1, dst01
289 bilinear_load_dst_8888 numpix, dst0, dst1, dst01
292 .macro bilinear_load_dst dst_fmt, op, numpix, dst0, dst1, dst01
293 bilinear_load_dst_&dst_fmt&_&op numpix, dst0, dst1, dst01
297 * Macros for duplicating partially loaded mask to fill entire register.
298 * We will apply mask to interleaved source pixels, that is
299 * (r0, r1, r2, r3, g0, g1, g2, g3) x (m0, m1, m2, m3, m0, m1, m2, m3)
300 * (b0, b1, b2, b3, a0, a1, a2, a3) x (m0, m1, m2, m3, m0, m1, m2, m3)
301 * So, we need to duplicate loaded mask into whole register.
304 * (r0, r1, x, x, g0, g1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
305 * (b0, b1, x, x, a0, a1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
306 * We can do some optimizations for this including one pixel cases.
308 .macro bilinear_duplicate_mask_x numpix, mask
311 .macro bilinear_duplicate_mask_8 numpix, mask
313 vdup.32 mask, mask[0]
315 vdup.16 mask, mask[0]
319 .error bilinear_duplicate_mask_8 is unsupported
323 .macro bilinear_duplicate_mask mask_fmt, numpix, mask
324 bilinear_duplicate_mask_&mask_fmt numpix, mask
328 * Macros for interleaving src and dst pixels to rrrr gggg bbbb aaaa form.
329 * Interleave should be done when maks is enabled or operator is 'over'.
331 .macro bilinear_interleave src0, src1, dst0, dst1
338 .macro bilinear_interleave_src_dst_x_src \
339 numpix, src0, src1, src01, dst0, dst1, dst01
342 .macro bilinear_interleave_src_dst_x_over \
343 numpix, src0, src1, src01, dst0, dst1, dst01
345 bilinear_interleave src0, src1, dst0, dst1
348 .macro bilinear_interleave_src_dst_x_add \
349 numpix, src0, src1, src01, dst0, dst1, dst01
352 .macro bilinear_interleave_src_dst_8_src \
353 numpix, src0, src1, src01, dst0, dst1, dst01
355 bilinear_interleave src0, src1, dst0, dst1
358 .macro bilinear_interleave_src_dst_8_over \
359 numpix, src0, src1, src01, dst0, dst1, dst01
361 bilinear_interleave src0, src1, dst0, dst1
364 .macro bilinear_interleave_src_dst_8_add \
365 numpix, src0, src1, src01, dst0, dst1, dst01
367 bilinear_interleave src0, src1, dst0, dst1
370 .macro bilinear_interleave_src_dst \
371 mask_fmt, op, numpix, src0, src1, src01, dst0, dst1, dst01
373 bilinear_interleave_src_dst_&mask_fmt&_&op \
374 numpix, src0, src1, src01, dst0, dst1, dst01
379 * Macros for applying masks to src pixels. (see combine_mask_u() function)
380 * src, dst should be in interleaved form.
381 * mask register should be in form (m0, m1, m2, m3).
383 .macro bilinear_apply_mask_to_src_x \
384 numpix, src0, src1, src01, mask, \
385 tmp01, tmp23, tmp45, tmp67
388 .macro bilinear_apply_mask_to_src_8 \
389 numpix, src0, src1, src01, mask, \
390 tmp01, tmp23, tmp45, tmp67
392 vmull.u8 tmp01, src0, mask
393 vmull.u8 tmp23, src1, mask
395 vrshr.u16 tmp45, tmp01, #8
396 vrshr.u16 tmp67, tmp23, #8
398 vraddhn.u16 src0, tmp45, tmp01
399 vraddhn.u16 src1, tmp67, tmp23
402 .macro bilinear_apply_mask_to_src \
403 mask_fmt, numpix, src0, src1, src01, mask, \
404 tmp01, tmp23, tmp45, tmp67
406 bilinear_apply_mask_to_src_&mask_fmt \
407 numpix, src0, src1, src01, mask, \
408 tmp01, tmp23, tmp45, tmp67
413 * Macros for combining src and destination pixels.
414 * Interleave or not is depending on operator 'op'.
416 .macro bilinear_combine_src \
417 numpix, src0, src1, src01, dst0, dst1, dst01, \
418 tmp01, tmp23, tmp45, tmp67, tmp8
421 .macro bilinear_combine_over \
422 numpix, src0, src1, src01, dst0, dst1, dst01, \
423 tmp01, tmp23, tmp45, tmp67, tmp8
425 vdup.32 tmp8, src1[1]
429 vmull.u8 tmp01, dst0, tmp8
431 vmull.u8 tmp23, dst1, tmp8
433 vrshr.u16 tmp45, tmp01, #8
434 vrshr.u16 tmp67, tmp23, #8
436 vraddhn.u16 dst0, tmp45, tmp01
437 vraddhn.u16 dst1, tmp67, tmp23
439 vqadd.u8 src01, dst01, src01
442 .macro bilinear_combine_add \
443 numpix, src0, src1, src01, dst0, dst1, dst01, \
444 tmp01, tmp23, tmp45, tmp67, tmp8
446 vqadd.u8 src01, dst01, src01
449 .macro bilinear_combine \
450 op, numpix, src0, src1, src01, dst0, dst1, dst01, \
451 tmp01, tmp23, tmp45, tmp67, tmp8
453 bilinear_combine_&op \
454 numpix, src0, src1, src01, dst0, dst1, dst01, \
455 tmp01, tmp23, tmp45, tmp67, tmp8
459 * Macros for final deinterleaving of destination pixels if needed.
461 .macro bilinear_deinterleave numpix, dst0, dst1, dst01
467 .macro bilinear_deinterleave_dst_x_src numpix, dst0, dst1, dst01
470 .macro bilinear_deinterleave_dst_x_over numpix, dst0, dst1, dst01
471 bilinear_deinterleave numpix, dst0, dst1, dst01
474 .macro bilinear_deinterleave_dst_x_add numpix, dst0, dst1, dst01
477 .macro bilinear_deinterleave_dst_8_src numpix, dst0, dst1, dst01
478 bilinear_deinterleave numpix, dst0, dst1, dst01
481 .macro bilinear_deinterleave_dst_8_over numpix, dst0, dst1, dst01
482 bilinear_deinterleave numpix, dst0, dst1, dst01
485 .macro bilinear_deinterleave_dst_8_add numpix, dst0, dst1, dst01
486 bilinear_deinterleave numpix, dst0, dst1, dst01
489 .macro bilinear_deinterleave_dst mask_fmt, op, numpix, dst0, dst1, dst01
490 bilinear_deinterleave_dst_&mask_fmt&_&op numpix, dst0, dst1, dst01
494 .macro bilinear_interpolate_last_pixel src_fmt, mask_fmt, dst_fmt, op
495 bilinear_load_&src_fmt d0, d1, d2
496 bilinear_load_mask mask_fmt, 1, d4
497 bilinear_load_dst dst_fmt, op, 1, d18, d19, q9
500 vshr.u16 d30, d24, #8
501 /* 4 cycles bubble */
503 vmlsl.u16 q0, d2, d30
504 vmlal.u16 q0, d3, d30
505 /* 5 cycles bubble */
506 bilinear_duplicate_mask mask_fmt, 1, d4
507 vshrn.u32 d0, q0, #16
508 /* 3 cycles bubble */
511 bilinear_interleave_src_dst \
512 mask_fmt, op, 1, d0, d1, q0, d18, d19, q9
513 bilinear_apply_mask_to_src \
514 mask_fmt, 1, d0, d1, q0, d4, \
517 op, 1, d0, d1, q0, d18, d19, q9, \
519 bilinear_deinterleave_dst mask_fmt, op, 1, d0, d1, q0
520 bilinear_store_&dst_fmt 1, q2, q3
523 .macro bilinear_interpolate_two_pixels src_fmt, mask_fmt, dst_fmt, op
524 bilinear_load_and_vertical_interpolate_two_&src_fmt \
525 q1, q11, d0, d1, d20, d21, d22, d23
526 bilinear_load_mask mask_fmt, 2, d4
527 bilinear_load_dst dst_fmt, op, 2, d18, d19, q9
528 vshr.u16 q15, q12, #8
529 vadd.u16 q12, q12, q13
531 vmlsl.u16 q0, d2, d30
532 vmlal.u16 q0, d3, d30
533 vshll.u16 q10, d22, #8
534 vmlsl.u16 q10, d22, d31
535 vmlal.u16 q10, d23, d31
536 vshrn.u32 d30, q0, #16
537 vshrn.u32 d31, q10, #16
538 bilinear_duplicate_mask mask_fmt, 2, d4
540 bilinear_interleave_src_dst \
541 mask_fmt, op, 2, d0, d1, q0, d18, d19, q9
542 bilinear_apply_mask_to_src \
543 mask_fmt, 2, d0, d1, q0, d4, \
546 op, 2, d0, d1, q0, d18, d19, q9, \
548 bilinear_deinterleave_dst mask_fmt, op, 2, d0, d1, q0
549 bilinear_store_&dst_fmt 2, q2, q3
552 .macro bilinear_interpolate_four_pixels src_fmt, mask_fmt, dst_fmt, op
553 bilinear_load_and_vertical_interpolate_four_&src_fmt \
554 q1, q11, d0, d1, d20, d21, d22, d23 \
555 q3, q9, d4, d5, d16, d17, d18, d19
557 vshr.u16 q15, q12, #8
558 vadd.u16 q12, q12, q13
560 vmlsl.u16 q0, d2, d30
561 vmlal.u16 q0, d3, d30
562 vshll.u16 q10, d22, #8
563 vmlsl.u16 q10, d22, d31
564 vmlal.u16 q10, d23, d31
565 vshr.u16 q15, q12, #8
567 vmlsl.u16 q2, d6, d30
568 vmlal.u16 q2, d7, d30
569 vshll.u16 q8, d18, #8
570 bilinear_load_mask mask_fmt, 4, d30
571 bilinear_load_dst dst_fmt, op, 4, d2, d3, q1
573 vmlsl.u16 q8, d18, d31
574 vmlal.u16 q8, d19, d31
575 vadd.u16 q12, q12, q13
576 vshrn.u32 d0, q0, #16
577 vshrn.u32 d1, q10, #16
578 vshrn.u32 d4, q2, #16
579 vshrn.u32 d5, q8, #16
580 bilinear_duplicate_mask mask_fmt, 4, d30
583 bilinear_interleave_src_dst \
584 mask_fmt, op, 4, d0, d1, q0, d2, d3, q1
585 bilinear_apply_mask_to_src \
586 mask_fmt, 4, d0, d1, q0, d30, \
589 op, 4, d0, d1, q0, d2, d3, q1, \
591 bilinear_deinterleave_dst mask_fmt, op, 4, d0, d1, q0
592 bilinear_store_&dst_fmt 4, q2, q3
595 .macro generate_bilinear_scanline_func_src_dst \
596 fname, src_fmt, dst_fmt, op, \
597 bpp_shift, prefetch_distance
599 pixman_asm_function fname
615 push {r4, r5, r6, r7, r8, r9}
616 mov PF_OFFS, #prefetch_distance
617 ldmia ip, {WB, X, UX, WIDTH}
618 mul PF_OFFS, PF_OFFS, UX
627 vadd.u16 d25, d25, d26
628 vadd.u16 q13, q13, q13
630 subs WIDTH, WIDTH, #4
632 mov PF_OFFS, PF_OFFS, asr #(16 - bpp_shift)
634 bilinear_interpolate_four_pixels src_fmt, x, dst_fmt, op
635 subs WIDTH, WIDTH, #4
640 bilinear_interpolate_two_pixels src_fmt, x, dst_fmt, op
644 bilinear_interpolate_last_pixel src_fmt, x, dst_fmt, op
646 pop {r4, r5, r6, r7, r8, r9}
666 .macro generate_bilinear_scanline_func_src_a8_dst \
667 fname, src_fmt, dst_fmt, op, \
668 bpp_shift, prefetch_distance
670 pixman_asm_function fname
687 push {r4, r5, r6, r7, r8, r9, r10, ip}
688 mov PF_OFFS, #prefetch_distance
689 ldmia ip, {WT, WB, X, UX, WIDTH}
690 mul PF_OFFS, PF_OFFS, UX
699 vadd.u16 d25, d25, d26
700 vadd.u16 q13, q13, q13
702 subs WIDTH, WIDTH, #4
704 mov PF_OFFS, PF_OFFS, asr #(16 - bpp_shift)
706 bilinear_interpolate_four_pixels src_fmt, 8, dst_fmt, op
707 subs WIDTH, WIDTH, #4
712 bilinear_interpolate_two_pixels src_fmt, 8, dst_fmt, op
716 bilinear_interpolate_last_pixel src_fmt, 8, dst_fmt, op
718 pop {r4, r5, r6, r7, r8, r9, r10, ip}
739 generate_bilinear_scanline_func_src_dst \
740 pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_neon, \
741 8888, 8888, over, 2, 28
743 generate_bilinear_scanline_func_src_dst \
744 pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_neon, \
745 8888, 8888, add, 2, 28
747 generate_bilinear_scanline_func_src_a8_dst \
748 pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_neon, \
749 8888, 8888, src, 2, 28
751 generate_bilinear_scanline_func_src_a8_dst \
752 pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_neon, \
753 8888, 0565, src, 2, 28
755 generate_bilinear_scanline_func_src_a8_dst \
756 pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_neon, \
757 0565, 8888, src, 1, 28
759 generate_bilinear_scanline_func_src_a8_dst \
760 pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_neon, \
761 0565, 0565, src, 1, 28
763 generate_bilinear_scanline_func_src_a8_dst \
764 pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_neon, \
765 8888, 8888, over, 2, 28
767 generate_bilinear_scanline_func_src_a8_dst \
768 pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_neon, \
769 8888, 8888, add, 2, 28