if (1 == inc_x)
{
-#ifdef ENABLE_PREFETCH
- FLOAT *x_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
+ if (n > 31)
{
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- x_pref = x + pref_offset + 128;
-#endif
+ FLOAT *x_pref;
+ BLASLONG pref_offset;
- for (i = (n >> 5); i--;)
- {
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 128(%[x_pref])\n\t"
- "pref 0, 160(%[x_pref])\n\t"
- "pref 0, 192(%[x_pref])\n\t"
- "pref 0, 224(%[x_pref])\n\t"
-
- : : [x_pref] "r" (x_pref)
- );
-
- x_pref += 64;
-#endif
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 128 + 32;
LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 5) - 1; i--;)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(x_pref, 128);
+ PREF_OFFSET(x_pref, 160);
+ PREF_OFFSET(x_pref, 192);
+ PREF_OFFSET(x_pref, 224);
+ x_pref += 64;
+
+ LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_W(src0);
+ sum_abs1 += AND_VEC_W(src1);
+ sum_abs2 += AND_VEC_W(src2);
+ sum_abs3 += AND_VEC_W(src3);
+ sum_abs0 += AND_VEC_W(src4);
+ sum_abs1 += AND_VEC_W(src5);
+ sum_abs2 += AND_VEC_W(src6);
+ sum_abs3 += AND_VEC_W(src7);
+
+ LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_W(src8);
+ sum_abs1 += AND_VEC_W(src9);
+ sum_abs2 += AND_VEC_W(src10);
+ sum_abs3 += AND_VEC_W(src11);
+ sum_abs0 += AND_VEC_W(src12);
+ sum_abs1 += AND_VEC_W(src13);
+ sum_abs2 += AND_VEC_W(src14);
+ sum_abs3 += AND_VEC_W(src15);
+ }
+
LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_W(src0);
{
inc_x2 = 2 * inc_x;
- for (i = (n >> 4); i--;)
+ if (n > 16)
{
LD_SP8_INC(x, inc_x2, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 4) - 1; i--;)
+ {
+ LD_SP8_INC(x, inc_x2, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_W(src0);
+ sum_abs1 += AND_VEC_W(src1);
+ sum_abs2 += AND_VEC_W(src2);
+ sum_abs3 += AND_VEC_W(src3);
+ sum_abs0 += AND_VEC_W(src4);
+ sum_abs1 += AND_VEC_W(src5);
+ sum_abs2 += AND_VEC_W(src6);
+ sum_abs3 += AND_VEC_W(src7);
+
+ LD_SP8_INC(x, inc_x2, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_W(src8);
+ sum_abs1 += AND_VEC_W(src9);
+ sum_abs2 += AND_VEC_W(src10);
+ sum_abs3 += AND_VEC_W(src11);
+ sum_abs0 += AND_VEC_W(src12);
+ sum_abs1 += AND_VEC_W(src13);
+ sum_abs2 += AND_VEC_W(src14);
+ sum_abs3 += AND_VEC_W(src15);
+ }
+
LD_SP8_INC(x, inc_x2, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_W(src0);
BLASLONG i = 0;
FLOAT dot[2];
BLASLONG inc_x2, inc_y2;
- FLOAT x0, x1, x2, x3, x4, x5, x6, x7;
- FLOAT y0, y1, y2, y3, y4, y5, y6, y7;
- v4f32 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7;
- v4f32 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7;
+ FLOAT x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, y6, y7;
+ v4f32 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7, vx8, vx9, vx10, vx11;
+ v4f32 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7, vy8, vy9, vy10, vy11;
v4f32 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i;
v4f32 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i;
v4f32 dot0 = {0, 0, 0, 0};
if ((1 == inc_x) && (1 == inc_y))
{
-#ifdef ENABLE_PREFETCH
- FLOAT *x_pref, *y_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
+ if (n > 15)
{
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- x_pref = x + pref_offset + 64;
+ FLOAT *x_pref, *y_pref;
+ BLASLONG pref_offset;
- pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
- {
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- y_pref = y + pref_offset + 64;
-#endif
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 64 + 16;
- for (i = (n >> 4); i--;)
- {
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 0(%[y_pref])\n\t"
- "pref 0, 32(%[y_pref])\n\t"
- "pref 0, 64(%[y_pref])\n\t"
- "pref 0, 96(%[y_pref])\n\t"
-
- : : [x_pref] "r" (x_pref), [y_pref] "r" (y_pref)
- );
-
- x_pref += 32;
- y_pref += 32;
-#endif
+ pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ y_pref = y + pref_offset + 64 + 16;
- LD_SP8_INC(x, 4, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
- LD_SP8_INC(y, 4, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
+ LD_SP4_INC(x, 4, vx0, vx1, vx2, vx3);
+ LD_SP4_INC(y, 4, vy0, vy1, vy2, vy3);
PCKEVOD_W2_SP(vx1, vx0, vx0r, vx0i);
- PCKEVOD_W2_SP(vx3, vx2, vx1r, vx1i);
- PCKEVOD_W2_SP(vx5, vx4, vx2r, vx2i);
- PCKEVOD_W2_SP(vx7, vx6, vx3r, vx3i);
-
PCKEVOD_W2_SP(vy1, vy0, vy0r, vy0i);
- PCKEVOD_W2_SP(vy3, vy2, vy1r, vy1i);
- PCKEVOD_W2_SP(vy5, vy4, vy2r, vy2i);
- PCKEVOD_W2_SP(vy7, vy6, vy3r, vy3i);
- dot0 += (vx0r * vy0r);
- dot0 OP1 (vx0i * vy0i);
- dot1 OP2 (vx0i * vy0r);
- dot1 += (vx0r * vy0i);
+ for (i = (n >> 4) - 1; i--;)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(y_pref, 0);
+ PREF_OFFSET(y_pref, 32);
+ PREF_OFFSET(y_pref, 64);
+ PREF_OFFSET(y_pref, 96);
+ x_pref += 32;
+ y_pref += 32;
+
+ vx4 = LD_SP(x); x += 4;
+ vx1r = (v4f32) __msa_pckev_w((v4i32) vx3, (v4i32) vx2);
+ dot0 += (vx0r * vy0r);
+ vx5 = LD_SP(x); x += 4;
+ vx1i = (v4f32) __msa_pckod_w((v4i32) vx3, (v4i32) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_SP(y); y += 4;
+ vy1r = (v4f32) __msa_pckev_w((v4i32) vy3, (v4i32) vy2);
+ dot2 += (vx1r * vy1r);
+ vy5 = LD_SP(y); y += 4;
+ vy1i = (v4f32) __msa_pckod_w((v4i32) vy3, (v4i32) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_SP(x); x += 4;
+ vx7 = LD_SP(x); x += 4;
+ vy6 = LD_SP(y); y += 4;
+ vy7 = LD_SP(y); y += 4;
+ vx8 = LD_SP(x); x += 4;
+ dot0 -= (vx0i * vy0i);
+ vx9 = LD_SP(x); x += 4;
+ vx2r = (v4f32) __msa_pckev_w((v4i32) vx5, (v4i32) vx4);
+ dot1 += (vx0r * vy0i);
+ vy8 = LD_SP(y); y += 4;
+ vx2i = (v4f32) __msa_pckod_w((v4i32) vx5, (v4i32) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy9 = LD_SP(y); y += 4;
+ vy2r = (v4f32) __msa_pckev_w((v4i32) vy5, (v4i32) vy4);
+ dot3 += (vx1r * vy1i);
+ vx10 = LD_SP(x); x += 4;
+ vy2i = (v4f32) __msa_pckod_w((v4i32) vy5, (v4i32) vy4);
+ vx11 = LD_SP(x); x += 4;
+ vx3r = (v4f32) __msa_pckev_w((v4i32) vx7, (v4i32) vx6);
+ dot4 += (vx2r * vy2r);
+ vy10 = LD_SP(y); y += 4;
+ vx3i = (v4f32) __msa_pckod_w((v4i32) vx7, (v4i32) vx6);
+ dot5 += (vx2i * vy2r);
+ vy11 = LD_SP(y); y += 4;
+ vy3r = (v4f32) __msa_pckev_w((v4i32) vy7, (v4i32) vy6);
+ vy3i = (v4f32) __msa_pckod_w((v4i32) vy7, (v4i32) vy6);
+ dot6 += (vx3r * vy3r);
+ vx0r = (v4f32) __msa_pckev_w((v4i32) vx9, (v4i32) vx8);
+ dot7 += (vx3i * vy3r);
+ vx0i = (v4f32) __msa_pckod_w((v4i32) vx9, (v4i32) vx8);
+ vy0r = (v4f32) __msa_pckev_w((v4i32) vy9, (v4i32) vy8);
+ vx2 = vx10;
+ vy0i = (v4f32) __msa_pckod_w((v4i32) vy9, (v4i32) vy8);
+ vx3 = vx11;
+ dot4 -= (vx2i * vy2i);
+ vy2 = vy10;
+ dot5 += (vx2r * vy2i);
+ vy3 = vy11;
+ dot6 -= (vx3i * vy3i);
+ dot7 += (vx3r * vy3i);
+ }
+ vx4 = LD_SP(x); x += 4;
+ vx1r = (v4f32) __msa_pckev_w((v4i32) vx3, (v4i32) vx2);
+ dot0 += (vx0r * vy0r);
+ vx5 = LD_SP(x); x += 4;
+ vx1i = (v4f32) __msa_pckod_w((v4i32) vx3, (v4i32) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_SP(y); y += 4;
+ vy1r = (v4f32) __msa_pckev_w((v4i32) vy3, (v4i32) vy2);
dot2 += (vx1r * vy1r);
- dot2 OP1 (vx1i * vy1i);
- dot3 OP2 (vx1i * vy1r);
+ vy5 = LD_SP(y); y += 4;
+ vy1i = (v4f32) __msa_pckod_w((v4i32) vy3, (v4i32) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_SP(x); x += 4;
+ vx7 = LD_SP(x); x += 4;
+ vy6 = LD_SP(y); y += 4;
+ vy7 = LD_SP(y); y += 4;
+ dot0 -= (vx0i * vy0i);
+ vx2r = (v4f32) __msa_pckev_w((v4i32) vx5, (v4i32) vx4);
+ dot1 += (vx0r * vy0i);
+ vx2i = (v4f32) __msa_pckod_w((v4i32) vx5, (v4i32) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy2r = (v4f32) __msa_pckev_w((v4i32) vy5, (v4i32) vy4);
dot3 += (vx1r * vy1i);
-
+ vy2i = (v4f32) __msa_pckod_w((v4i32) vy5, (v4i32) vy4);
+ vx3r = (v4f32) __msa_pckev_w((v4i32) vx7, (v4i32) vx6);
dot4 += (vx2r * vy2r);
- dot4 OP1 (vx2i * vy2i);
- dot5 OP2 (vx2i * vy2r);
- dot5 += (vx2r * vy2i);
-
+ vx3i = (v4f32) __msa_pckod_w((v4i32) vx7, (v4i32) vx6);
+ dot5 += (vx2i * vy2r);
+ vy3r = (v4f32) __msa_pckev_w((v4i32) vy7, (v4i32) vy6);
+ vy3i = (v4f32) __msa_pckod_w((v4i32) vy7, (v4i32) vy6);
dot6 += (vx3r * vy3r);
- dot6 OP1 (vx3i * vy3i);
- dot7 OP2 (vx3i * vy3r);
+ dot7 += (vx3i * vy3r);
+ dot4 -= (vx2i * vy2i);
+ dot5 += (vx2r * vy2i);
+ dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}
if (1 == inc_x)
{
-#ifdef ENABLE_PREFETCH
- FLOAT *x_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
+ if (n > 31)
{
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- x_pref = x + pref_offset + 64;
-#endif
+ FLOAT *x_pref;
+ BLASLONG pref_offset;
- for (i = (n >> 5); i--;)
- {
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 128(%[x_pref])\n\t"
- "pref 0, 160(%[x_pref])\n\t"
- "pref 0, 192(%[x_pref])\n\t"
- "pref 0, 224(%[x_pref])\n\t"
-
- : : [x_pref] "r" (x_pref)
- );
-
- x_pref += 32;
-#endif
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 64 + 16;
LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 5) - 1; i--;)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(x_pref, 128);
+ PREF_OFFSET(x_pref, 160);
+ PREF_OFFSET(x_pref, 192);
+ PREF_OFFSET(x_pref, 224);
+ x_pref += 32;
+
+ LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_D(src0);
+ sum_abs1 += AND_VEC_D(src1);
+ sum_abs2 += AND_VEC_D(src2);
+ sum_abs3 += AND_VEC_D(src3);
+ sum_abs0 += AND_VEC_D(src4);
+ sum_abs1 += AND_VEC_D(src5);
+ sum_abs2 += AND_VEC_D(src6);
+ sum_abs3 += AND_VEC_D(src7);
+
+ LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_D(src8);
+ sum_abs1 += AND_VEC_D(src9);
+ sum_abs2 += AND_VEC_D(src10);
+ sum_abs3 += AND_VEC_D(src11);
+ sum_abs0 += AND_VEC_D(src12);
+ sum_abs1 += AND_VEC_D(src13);
+ sum_abs2 += AND_VEC_D(src14);
+ sum_abs3 += AND_VEC_D(src15);
+ }
+
LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_D(src0);
}
else
{
- for (i = (n >> 4); i--;)
+ if (n > 16)
{
LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 4) - 1; i--;)
+ {
+ LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_D(src0);
+ sum_abs1 += AND_VEC_D(src1);
+ sum_abs2 += AND_VEC_D(src2);
+ sum_abs3 += AND_VEC_D(src3);
+ sum_abs0 += AND_VEC_D(src4);
+ sum_abs1 += AND_VEC_D(src5);
+ sum_abs2 += AND_VEC_D(src6);
+ sum_abs3 += AND_VEC_D(src7);
+
+ LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_D(src8);
+ sum_abs1 += AND_VEC_D(src9);
+ sum_abs2 += AND_VEC_D(src10);
+ sum_abs3 += AND_VEC_D(src11);
+ sum_abs0 += AND_VEC_D(src12);
+ sum_abs1 += AND_VEC_D(src13);
+ sum_abs2 += AND_VEC_D(src14);
+ sum_abs3 += AND_VEC_D(src15);
+ }
+
LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_D(src0);
if ((1 == inc_x) && (1 == inc_y))
{
+ FLOAT *x_pref, *y_pref;
+ BLASLONG pref_offset;
+
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 32;
+
+ pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ y_pref = y + pref_offset + 32;
+
for (i = (n >> 4); i--;)
{
LD_DP8_INC(x, 2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_DP8_INC(y, 2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 256(%[x])\n\t"
- "pref 0, 288(%[x])\n\t"
- "pref 0, 320(%[x])\n\t"
- "pref 0, 352(%[x])\n\t"
- "pref 0, 256(%[y])\n\t"
- "pref 0, 288(%[y])\n\t"
- "pref 0, 320(%[y])\n\t"
- "pref 0, 352(%[y])\n\t"
-
- : : [x] "r" (x), [y] "r" (y)
- );
-#endif
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(y_pref, 0);
+ PREF_OFFSET(y_pref, 32);
+ PREF_OFFSET(y_pref, 64);
+ PREF_OFFSET(y_pref, 96);
+ x_pref += 16;
+ y_pref += 16;
dot0 += (vy0 * vx0);
dot1 += (vy1 * vx1);
FLOAT *c_nxt2line = c + 2 * ldc;
FLOAT *c_nxt3line = c + 3 * ldc;
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, -96(%[a]) \n\t"
- "pref 0, -32(%[a]) \n\t"
- "pref 0, -160(%[a]) \n\t"
- "pref 0, -224(%[a]) \n\t"
- "pref 0, -64(%[a]) \n\t"
- "pref 0, -128(%[a]) \n\t"
- "pref 0, -192(%[a]) \n\t"
- "pref 0, -256(%[a]) \n\t"
- "pref 0, -320(%[a]) \n\t"
- "pref 0, -384(%[a]) \n\t"
- "pref 0, -448(%[a]) \n\t"
- "pref 0, -512(%[a]) \n\t"
-
- :
- : [a] "r"(a)
- );
-#endif
+ PREF_OFFSET(a, -96);
+ PREF_OFFSET(a, -32);
+ PREF_OFFSET(a, -160);
+ PREF_OFFSET(a, -224);
+ PREF_OFFSET(a, -64);
+ PREF_OFFSET(a, -128);
+ PREF_OFFSET(a, -192);
+ PREF_OFFSET(a, -256);
+ PREF_OFFSET(a, -320);
+ PREF_OFFSET(a, -384);
+ PREF_OFFSET(a, -448);
+ PREF_OFFSET(a, -512);
LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3);
LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7);
if (bk > 0)
{
- BLASLONG i;
- FLOAT *pba = a, *pbb = b;
- v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
+ BLASLONG i, pref_offset;
+ FLOAT *pba = a, *pbb = b, *pa0_pref;
+ v2f64 src_b, src_b0, src_b1;
- LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
- LD_DP2_INC(pbb, 2, src_b0, src_b1);
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
- for (i = (bk - 1) >> 1; i--;)
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 128(%[pba]) \n\t"
- "pref 0, 160(%[pba]) \n\t"
- "pref 0, 192(%[pba]) \n\t"
- "pref 0, 224(%[pba]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+
+ pa0_pref = a + pref_offset;
- :
- : [pba] "r"(pba)
- );
-#endif
+ for (i = bk >> 1; i--;)
+ {
+ PREF_OFFSET(pa0_pref, 128);
+ PREF_OFFSET(pa0_pref, 160);
+ PREF_OFFSET(pa0_pref, 192);
+ PREF_OFFSET(pa0_pref, 224);
- LD_DP4_INC(pba, 2, src_a8, src_a9, src_a16, src_a17);
- LD_DP2_INC(pbb, 2, src_b2, src_b3);
+ LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
+ LD_DP2_INC(pbb, 2, src_b0, src_b1);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
- src_c0 -= src_a8 * src_b;
- src_c1 -= src_a9 * src_b;
- src_c2 -= src_a16 * src_b;
- src_c3 -= src_a17 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
- src_c4 -= src_a8 * src_b;
- src_c5 -= src_a9 * src_b;
- src_c6 -= src_a16 * src_b;
- src_c7 -= src_a17 * src_b;
-
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
- src_c8 -= src_a8 * src_b;
- src_c9 -= src_a9 * src_b;
- src_c10 -= src_a16 * src_b;
- src_c11 -= src_a17 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
- src_c12 -= src_a8 * src_b;
- src_c13 -= src_a9 * src_b;
- src_c14 -= src_a16 * src_b;
- src_c15 -= src_a17 * src_b;
- }
-
- if ((bk - 1) & 1)
- {
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
+ pa0_pref += 16;
+ }
+
+ if (bk & 1)
+ {
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
- }
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
- src_c0 -= src_a0 * src_b;
- src_c1 -= src_a1 * src_b;
- src_c2 -= src_a2 * src_b;
- src_c3 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c0 -= src_a0 * src_b;
+ src_c1 -= src_a1 * src_b;
+ src_c2 -= src_a2 * src_b;
+ src_c3 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
- src_c4 -= src_a0 * src_b;
- src_c5 -= src_a1 * src_b;
- src_c6 -= src_a2 * src_b;
- src_c7 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c4 -= src_a0 * src_b;
+ src_c5 -= src_a1 * src_b;
+ src_c6 -= src_a2 * src_b;
+ src_c7 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
- src_c8 -= src_a0 * src_b;
- src_c9 -= src_a1 * src_b;
- src_c10 -= src_a2 * src_b;
- src_c11 -= src_a3 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
- src_c12 -= src_a0 * src_b;
- src_c13 -= src_a1 * src_b;
- src_c14 -= src_a2 * src_b;
- src_c15 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c8 -= src_a0 * src_b;
+ src_c9 -= src_a1 * src_b;
+ src_c10 -= src_a2 * src_b;
+ src_c11 -= src_a3 * src_b;
+
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c12 -= src_a0 * src_b;
+ src_c13 -= src_a1 * src_b;
+ src_c14 -= src_a2 * src_b;
+ src_c15 -= src_a3 * src_b;
+ }
}
a -= 64;
FLOAT *c_nxt2line = c + 2 * ldc;
FLOAT *c_nxt3line = c + 3 * ldc;
-#ifdef ENABLE_PREFETCH
a += bk * 8;
- __asm__ __volatile__(
- "pref 0, (%[a]) \n\t"
- "pref 0, 32(%[a]) \n\t"
- "pref 0, 72(%[a]) \n\t"
- "pref 0, 104(%[a]) \n\t"
- "pref 0, 144(%[a]) \n\t"
- "pref 0, 176(%[a]) \n\t"
- "pref 0, 216(%[a]) \n\t"
- "pref 0, 248(%[a]) \n\t"
- "pref 0, 288(%[a]) \n\t"
- "pref 0, 360(%[a]) \n\t"
- "pref 0, 504(%[a]) \n\t"
- "pref 0, 432(%[a]) \n\t"
-
- :
- : [a] "r"(a)
- );
+ PREF_OFFSET(a, 0);
+ PREF_OFFSET(a, 32);
+ PREF_OFFSET(a, 72);
+ PREF_OFFSET(a, 104);
+ PREF_OFFSET(a, 144);
+ PREF_OFFSET(a, 176);
+ PREF_OFFSET(a, 216);
+ PREF_OFFSET(a, 248);
+ PREF_OFFSET(a, 288);
+ PREF_OFFSET(a, 360);
+ PREF_OFFSET(a, 504);
+ PREF_OFFSET(a, 432);
a -= bk * 8;
-#endif
LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3);
LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7);
if (bk)
{
- BLASLONG i;
- v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
+ BLASLONG i, pref_offset;
+ FLOAT *pa0_pref;
+ v2f64 src_b, src_b0, src_b1;
- LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
- LD_DP2_INC(b, 2, src_b0, src_b1);
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
- for (i = ((bk - 1) >> 1); i--;)
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 128(%[a]) \n\t"
- "pref 0, 160(%[a]) \n\t"
- "pref 0, 192(%[a]) \n\t"
- "pref 0, 224(%[a]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
- :
- : [a] "r"(a)
- );
-#endif
+ pa0_pref = a + pref_offset;
- LD_DP4_INC(a, 2, src_a4, src_a5, src_a6, src_a7);
- LD_DP2_INC(b, 2, src_b2, src_b3);
+ for (i = (bk >> 1); i--;)
+ {
+ PREF_OFFSET(pa0_pref, 128);
+ PREF_OFFSET(pa0_pref, 160);
+ PREF_OFFSET(pa0_pref, 192);
+ PREF_OFFSET(pa0_pref, 224);
+
+ LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
+ LD_DP2_INC(b, 2, src_b0, src_b1);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
- src_c0 -= src_a4 * src_b;
- src_c1 -= src_a5 * src_b;
- src_c2 -= src_a6 * src_b;
- src_c3 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
- src_c4 -= src_a4 * src_b;
- src_c5 -= src_a5 * src_b;
- src_c6 -= src_a6 * src_b;
- src_c7 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
- src_c8 -= src_a4 * src_b;
- src_c9 -= src_a5 * src_b;
- src_c10 -= src_a6 * src_b;
- src_c11 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
- src_c12 -= src_a4 * src_b;
- src_c13 -= src_a5 * src_b;
- src_c14 -= src_a6 * src_b;
- src_c15 -= src_a7 * src_b;
- }
-
- if ((bk - 1) & 1)
- {
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
+ pa0_pref += 16;
+ }
+
+ if (bk & 1)
+ {
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
- }
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
- src_c0 -= src_a0 * src_b;
- src_c1 -= src_a1 * src_b;
- src_c2 -= src_a2 * src_b;
- src_c3 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c0 -= src_a0 * src_b;
+ src_c1 -= src_a1 * src_b;
+ src_c2 -= src_a2 * src_b;
+ src_c3 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
- src_c4 -= src_a0 * src_b;
- src_c5 -= src_a1 * src_b;
- src_c6 -= src_a2 * src_b;
- src_c7 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c4 -= src_a0 * src_b;
+ src_c5 -= src_a1 * src_b;
+ src_c6 -= src_a2 * src_b;
+ src_c7 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
- src_c8 -= src_a0 * src_b;
- src_c9 -= src_a1 * src_b;
- src_c10 -= src_a2 * src_b;
- src_c11 -= src_a3 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
- src_c12 -= src_a0 * src_b;
- src_c13 -= src_a1 * src_b;
- src_c14 -= src_a2 * src_b;
- src_c15 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c8 -= src_a0 * src_b;
+ src_c9 -= src_a1 * src_b;
+ src_c10 -= src_a2 * src_b;
+ src_c11 -= src_a3 * src_b;
+
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c12 -= src_a0 * src_b;
+ src_c13 -= src_a1 * src_b;
+ src_c14 -= src_a2 * src_b;
+ src_c15 -= src_a3 * src_b;
+ }
}
ILVRL_D2_DP(src_c4, src_c0, res_c0, res_c1);
if (bk)
{
- BLASLONG i;
- v2f64 src_a0, src_a1, src_a2, src_a3, src_a4, src_a5, src_a6, src_a7;
- v2f64 src_b;
+ BLASLONG i, pref_offset;
+ FLOAT *pa0_pref;
+ v2f64 src_a0, src_a1, src_a2, src_a3, src_b;
- LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
- LD_DP2_INC(b, 2, src_b0, src_b1);
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
- for (i = ((bk - 1) >> 1); i--;)
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 128(%[a]) \n\t"
- "pref 0, 160(%[a]) \n\t"
- "pref 0, 192(%[a]) \n\t"
- "pref 0, 224(%[a]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+
+ pa0_pref = a + pref_offset;
- :
- : [a] "r"(a)
- );
-#endif
+ for (i = (bk >> 1); i--;)
+ {
+ PREF_OFFSET(pa0_pref, 128);
+ PREF_OFFSET(pa0_pref, 160);
+ PREF_OFFSET(pa0_pref, 192);
+ PREF_OFFSET(pa0_pref, 224);
- LD_DP4_INC(a, 2, src_a4, src_a5, src_a6, src_a7);
- LD_DP2_INC(b, 2, src_b2, src_b3);
+ LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
+ LD_DP2_INC(b, 2, src_b0, src_b1);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
- src_c0 -= src_a4 * src_b;
- src_c1 -= src_a5 * src_b;
- src_c2 -= src_a6 * src_b;
- src_c3 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
- src_c4 -= src_a4 * src_b;
- src_c5 -= src_a5 * src_b;
- src_c6 -= src_a6 * src_b;
- src_c7 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
- src_c8 -= src_a4 * src_b;
- src_c9 -= src_a5 * src_b;
- src_c10 -= src_a6 * src_b;
- src_c11 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
- src_c12 -= src_a4 * src_b;
- src_c13 -= src_a5 * src_b;
- src_c14 -= src_a6 * src_b;
- src_c15 -= src_a7 * src_b;
- }
-
- if ((bk - 1) & 1)
- {
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
+ pa0_pref += 16;
+ }
+
+ if (bk & 1)
+ {
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
- }
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
- src_c0 -= src_a0 * src_b;
- src_c1 -= src_a1 * src_b;
- src_c2 -= src_a2 * src_b;
- src_c3 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c0 -= src_a0 * src_b;
+ src_c1 -= src_a1 * src_b;
+ src_c2 -= src_a2 * src_b;
+ src_c3 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
- src_c4 -= src_a0 * src_b;
- src_c5 -= src_a1 * src_b;
- src_c6 -= src_a2 * src_b;
- src_c7 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c4 -= src_a0 * src_b;
+ src_c5 -= src_a1 * src_b;
+ src_c6 -= src_a2 * src_b;
+ src_c7 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
- src_c8 -= src_a0 * src_b;
- src_c9 -= src_a1 * src_b;
- src_c10 -= src_a2 * src_b;
- src_c11 -= src_a3 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
- src_c12 -= src_a0 * src_b;
- src_c13 -= src_a1 * src_b;
- src_c14 -= src_a2 * src_b;
- src_c15 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c8 -= src_a0 * src_b;
+ src_c9 -= src_a1 * src_b;
+ src_c10 -= src_a2 * src_b;
+ src_c11 -= src_a3 * src_b;
+
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c12 -= src_a0 * src_b;
+ src_c13 -= src_a1 * src_b;
+ src_c14 -= src_a2 * src_b;
+ src_c15 -= src_a3 * src_b;
+ }
}
src_b0 = LD_DP(b + 0);
if (bk > 0)
{
- BLASLONG i;
- FLOAT *pba = a, *pbb = b;
- v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
- v2f64 src_a0, src_a1, src_a2, src_a3, src_a4, src_a5, src_a6, src_a7;
+ BLASLONG i, pref_offset;
+ FLOAT *pba = a, *pbb = b, *pa0_pref;
+ v2f64 src_b, src_b0, src_b1, src_a0, src_a1, src_a2, src_a3;
+
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
+
+ if (pref_offset)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
- LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
- LD_DP2_INC(pbb, 2, src_b0, src_b1);
+ pa0_pref = a + pref_offset;
- for (i = ((bk - 1) >> 1); i--;)
+ for (i = (bk >> 1); i--;)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 128(%[pba]) \n\t"
- "pref 0, 160(%[pba]) \n\t"
- "pref 0, 192(%[pba]) \n\t"
- "pref 0, 224(%[pba]) \n\t"
-
- :
- : [pba] "r"(pba)
- );
-#endif
- LD_DP4_INC(pba, 2, src_a4, src_a5, src_a6, src_a7);
- LD_DP2_INC(pbb, 2, src_b2, src_b3);
+ PREF_OFFSET(pa0_pref, 128);
+ PREF_OFFSET(pa0_pref, 160);
+ PREF_OFFSET(pa0_pref, 192);
+ PREF_OFFSET(pa0_pref, 224);
+
+ LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
+ LD_DP2_INC(pbb, 2, src_b0, src_b1);
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
- src_c0 -= src_a4 * src_b;
- src_c1 -= src_a5 * src_b;
- src_c2 -= src_a6 * src_b;
- src_c3 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
- src_c4 -= src_a4 * src_b;
- src_c5 -= src_a5 * src_b;
- src_c6 -= src_a6 * src_b;
- src_c7 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
- src_c8 -= src_a4 * src_b;
- src_c9 -= src_a5 * src_b;
- src_c10 -= src_a6 * src_b;
- src_c11 -= src_a7 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
- src_c12 -= src_a4 * src_b;
- src_c13 -= src_a5 * src_b;
- src_c14 -= src_a6 * src_b;
- src_c15 -= src_a7 * src_b;
- }
-
- if ((bk - 1) & 1)
- {
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
+ pa0_pref += 16;
+ }
+
+ if (bk & 1)
+ {
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
- }
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
- src_c0 -= src_a0 * src_b;
- src_c1 -= src_a1 * src_b;
- src_c2 -= src_a2 * src_b;
- src_c3 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c0 -= src_a0 * src_b;
+ src_c1 -= src_a1 * src_b;
+ src_c2 -= src_a2 * src_b;
+ src_c3 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
- src_c4 -= src_a0 * src_b;
- src_c5 -= src_a1 * src_b;
- src_c6 -= src_a2 * src_b;
- src_c7 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
+ src_c4 -= src_a0 * src_b;
+ src_c5 -= src_a1 * src_b;
+ src_c6 -= src_a2 * src_b;
+ src_c7 -= src_a3 * src_b;
- src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
- src_c8 -= src_a0 * src_b;
- src_c9 -= src_a1 * src_b;
- src_c10 -= src_a2 * src_b;
- src_c11 -= src_a3 * src_b;
-
- src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
- src_c12 -= src_a0 * src_b;
- src_c13 -= src_a1 * src_b;
- src_c14 -= src_a2 * src_b;
- src_c15 -= src_a3 * src_b;
+ src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c8 -= src_a0 * src_b;
+ src_c9 -= src_a1 * src_b;
+ src_c10 -= src_a2 * src_b;
+ src_c11 -= src_a3 * src_b;
+
+ src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
+ src_c12 -= src_a0 * src_b;
+ src_c13 -= src_a1 * src_b;
+ src_c14 -= src_a2 * src_b;
+ src_c15 -= src_a3 * src_b;
+ }
}
a -= 32;
#define ENABLE_PREFETCH
+#ifdef ENABLE_PREFETCH
+inline static void prefetch_load_lf(unsigned char *src)
+{
+ __asm__ __volatile__("pref 0, 0(%[src]) \n\t" : : [src] "r" (src));
+}
+
+#define PREFETCH(PTR) prefetch_load_lf((unsigned char *)(PTR));
+
+#define STRNG(X) #X
+#define PREF_OFFSET(src_ptr, offset) \
+ __asm__ __volatile__("pref 0, " STRNG(offset) "(%[src]) \n\t" : : [src] "r" (src_ptr));
+
+#else
+#define PREFETCH(PTR)
+#define PREF_OFFSET(src_ptr, offset)
+#endif
+
#define LD_W(RTYPE, psrc) *((RTYPE *)(psrc))
#define LD_SP(...) LD_W(v4f32, __VA_ARGS__)
if (1 == inc_x)
{
-#ifdef ENABLE_PREFETCH
- FLOAT *x_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
+ if (n > 63)
{
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- x_pref = x + pref_offset + 128;
-#endif
+ FLOAT *x_pref;
+ BLASLONG pref_offset;
- for (i = 0; i < (n >> 6); i++)
- {
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 128(%[x_pref])\n\t"
- "pref 0, 160(%[x_pref])\n\t"
- "pref 0, 192(%[x_pref])\n\t"
- "pref 0, 224(%[x_pref])\n\t"
-
- : : [x_pref] "r" (x_pref)
- );
-
- x_pref += 64;
-#endif
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 128 + 32;
LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = 0; i < (n >> 6) - 1; i++)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(x_pref, 128);
+ PREF_OFFSET(x_pref, 160);
+ PREF_OFFSET(x_pref, 192);
+ PREF_OFFSET(x_pref, 224);
+ x_pref += 64;
+
+ LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_W(src0);
+ sum_abs1 += AND_VEC_W(src1);
+ sum_abs2 += AND_VEC_W(src2);
+ sum_abs3 += AND_VEC_W(src3);
+ sum_abs0 += AND_VEC_W(src4);
+ sum_abs1 += AND_VEC_W(src5);
+ sum_abs2 += AND_VEC_W(src6);
+ sum_abs3 += AND_VEC_W(src7);
+
+ LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_W(src8);
+ sum_abs1 += AND_VEC_W(src9);
+ sum_abs2 += AND_VEC_W(src10);
+ sum_abs3 += AND_VEC_W(src11);
+ sum_abs0 += AND_VEC_W(src12);
+ sum_abs1 += AND_VEC_W(src13);
+ sum_abs2 += AND_VEC_W(src14);
+ sum_abs3 += AND_VEC_W(src15);
+ }
+
LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_W(src0);
if ((1 == inc_x) && (1 == inc_y))
{
+ FLOAT *x_pref, *y_pref;
+ BLASLONG pref_offset;
+
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 64;
+
+ pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ y_pref = y + pref_offset + 64;
+
for (i = (n >> 5); i--;)
{
LD_SP8_INC(x, 4, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_SP8_INC(y, 4, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 256(%[x])\n\t"
- "pref 0, 288(%[x])\n\t"
- "pref 0, 320(%[x])\n\t"
- "pref 0, 352(%[x])\n\t"
- "pref 0, 256(%[y])\n\t"
- "pref 0, 288(%[y])\n\t"
- "pref 0, 320(%[y])\n\t"
- "pref 0, 352(%[y])\n\t"
-
- : : [x] "r" (x), [y] "r" (y)
- );
-#endif
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(y_pref, 0);
+ PREF_OFFSET(y_pref, 32);
+ PREF_OFFSET(y_pref, 64);
+ PREF_OFFSET(y_pref, 96);
+ x_pref += 32;
+ y_pref += 32;
dot0 += (vy0 * vx0);
dot1 += (vy1 * vx1);
if (bk > 0)
{
- BLASLONG k;
- FLOAT *aa = a, *bb = b;
- v4f32 src_bb0, src_bb1, src_b0, src_b1, src_b2, src_b3, src_a1;
+ BLASLONG k, pref_offset;
+ FLOAT *aa = a, *bb = b, *pa0_pref;
+ v4f32 src_a1, src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;
- for (k = 0; k < (bk >> 1); k++)
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
+
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 64(%[aa]) \n\t"
- "pref 0, 96(%[aa]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
- :
- : [aa] "r" (aa)
- );
-#endif
+ pa0_pref = a + pref_offset;
+
+ for (k = 0; k < (bk >> 1); k++)
+ {
+ PREF_OFFSET(pa0_pref, 64);
+ PREF_OFFSET(pa0_pref, 96);
LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1);
src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3;
+
+ pa0_pref += 16;
}
if (bk & 1)
{
- LD_SP2(aa, 4, src_a0, src_a1);
- LD_SP2(bb, 4, src_bb0, src_bb1);
+ LD_SP2_INC(aa, 4, src_a0, src_a1);
+ LD_SP2_INC(bb, 4, src_bb0, src_bb1);
SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3);
src_c0 -= src_a0 * src_b0;
if (bk > 0)
{
- BLASLONG k;
- v4f32 src_b, src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;
+ BLASLONG k, pref_offset;
+ FLOAT *pa0_pref;
+ v4f32 src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;
- for (k = 0; k < (bk >> 1); k++)
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
+
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 64(%[a]) \n\t"
- "pref 0, 96(%[a]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
- :
- : [a] "r" (a)
- );
-#endif
+ pa0_pref = a + pref_offset;
+
+ for (k = 0; k < (bk >> 1); k++)
+ {
+ PREF_OFFSET(pa0_pref, 64);
+ PREF_OFFSET(pa0_pref, 96);
LD_SP2_INC(a, 4, src_a0, src_a1);
LD_SP2_INC(b, 4, src_bb0, src_bb1);
src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3;
+
+ pa0_pref += 16;
}
if (bk & 1)
if (bk > 0)
{
- BLASLONG k;
+ BLASLONG k, pref_offset;
+ FLOAT *pa0_pref;
v4f32 src_a0, src_a1, src_bb0, src_bb1;
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
+
+ if (pref_offset)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+
+ pa0_pref = a + pref_offset;
+
for (k = 0; k < (bk >> 1); k++)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 64(%[a]) \n\t"
- "pref 0, 96(%[a]) \n\t"
-
- :
- : [a] "r" (a)
- );
-#endif
+ PREF_OFFSET(pa0_pref, 64);
+ PREF_OFFSET(pa0_pref, 96);
+
LD_SP2_INC(a, 4, src_a0, src_a1);
LD_SP2_INC(b, 4, src_bb0, src_bb1);
src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3;
+
+ pa0_pref += 16;
}
if (bk & 1)
if (bk > 0)
{
- BLASLONG k;
- FLOAT *aa = a, *bb = b;
+ BLASLONG k, pref_offset;
+ FLOAT *aa = a, *bb = b, *pa0_pref;
v4f32 src_a0, src_a1, src_b1, src_b2, src_b3, src_bb0, src_bb1;
- for (k = 0; k < (bk >> 1); k++)
+ pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);
+
+ if (pref_offset)
{
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 64(%[aa]) \n\t"
- "pref 0, 96(%[aa]) \n\t"
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
- :
- : [aa] "r" (aa)
- );
-#endif
+ pa0_pref = a + pref_offset;
+
+ for (k = 0; k < (bk >> 1); k++)
+ {
+ PREF_OFFSET(pa0_pref, 64);
+ PREF_OFFSET(pa0_pref, 96);
LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1);
src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3;
+
+ pa0_pref += 16;
}
if (bk & 1)
{
- LD_SP2(aa, 4, src_a0, src_a1);
- LD_SP2(bb, 4, src_bb0, src_bb1);
+ LD_SP2_INC(aa, 4, src_a0, src_a1);
+ LD_SP2_INC(bb, 4, src_bb0, src_bb1);
SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3);
src_c0 -= src_a0 * src_b0;
if (1 == inc_x)
{
-#ifdef ENABLE_PREFETCH
- FLOAT *x_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
+ if (n > 16)
{
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(FLOAT);
- x_pref = x + pref_offset + 64;
-#endif
+ FLOAT *x_pref;
+ BLASLONG pref_offset;
- for (i = (n >> 4); i--;)
- {
-#ifdef ENABLE_PREFETCH
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 128(%[x_pref])\n\t"
- "pref 0, 160(%[x_pref])\n\t"
- "pref 0, 192(%[x_pref])\n\t"
- "pref 0, 224(%[x_pref])\n\t"
-
- : : [x_pref] "r" (x_pref)
- );
-
- x_pref += 32;
-#endif
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 64 + 16;
LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 4) - 1; i--;)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(x_pref, 128);
+ PREF_OFFSET(x_pref, 160);
+ PREF_OFFSET(x_pref, 192);
+ PREF_OFFSET(x_pref, 224);
+ x_pref += 32;
+
+ LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_D(src0);
+ sum_abs1 += AND_VEC_D(src1);
+ sum_abs2 += AND_VEC_D(src2);
+ sum_abs3 += AND_VEC_D(src3);
+ sum_abs0 += AND_VEC_D(src4);
+ sum_abs1 += AND_VEC_D(src5);
+ sum_abs2 += AND_VEC_D(src6);
+ sum_abs3 += AND_VEC_D(src7);
+
+ LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_D(src8);
+ sum_abs1 += AND_VEC_D(src9);
+ sum_abs2 += AND_VEC_D(src10);
+ sum_abs3 += AND_VEC_D(src11);
+ sum_abs0 += AND_VEC_D(src12);
+ sum_abs1 += AND_VEC_D(src13);
+ sum_abs2 += AND_VEC_D(src14);
+ sum_abs3 += AND_VEC_D(src15);
+ }
+
LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_D(src0);
{
inc_x *= 2;
- for (i = (n >> 4); i--;)
+ if (n > 16)
{
LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
+ for (i = (n >> 4) - 1; i--;)
+ {
+ LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);
+
+ sum_abs0 += AND_VEC_D(src0);
+ sum_abs1 += AND_VEC_D(src1);
+ sum_abs2 += AND_VEC_D(src2);
+ sum_abs3 += AND_VEC_D(src3);
+ sum_abs0 += AND_VEC_D(src4);
+ sum_abs1 += AND_VEC_D(src5);
+ sum_abs2 += AND_VEC_D(src6);
+ sum_abs3 += AND_VEC_D(src7);
+
+ LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
+
+ sum_abs0 += AND_VEC_D(src8);
+ sum_abs1 += AND_VEC_D(src9);
+ sum_abs2 += AND_VEC_D(src10);
+ sum_abs3 += AND_VEC_D(src11);
+ sum_abs0 += AND_VEC_D(src12);
+ sum_abs1 += AND_VEC_D(src13);
+ sum_abs2 += AND_VEC_D(src14);
+ sum_abs3 += AND_VEC_D(src15);
+ }
+
LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);
sum_abs0 += AND_VEC_D(src0);
BLASLONG i = 0;
FLOAT dot[2];
BLASLONG inc_x2, inc_y2;
- v2f64 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7;
- v2f64 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7;
+ v2f64 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7, vx8, vx9, vx10, vx11;
+ v2f64 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7, vy8, vy9, vy10, vy11;
v2f64 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i;
v2f64 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i;
v2f64 dot0 = {0, 0};
inc_x2 = 2 * inc_x;
inc_y2 = 2 * inc_y;
-
-#ifdef ENABLE_PREFETCH
if ((1 == inc_x) && (1 == inc_y))
{
- double *x_pref, *y_pref;
- BLASLONG pref_offset;
-
- pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
- {
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(double);
- x_pref = x + pref_offset + 32;
-
- pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
- if (pref_offset > 0)
- {
- pref_offset = L1_DATA_LINESIZE - pref_offset;
- }
- pref_offset = pref_offset / sizeof(double);
- y_pref = y + pref_offset + 32;
-
- for (i = (n >> 3); i--;)
+ if (n > 7)
{
- __asm__ __volatile__(
- "pref 0, 0(%[x_pref])\n\t"
- "pref 0, 32(%[x_pref])\n\t"
- "pref 0, 64(%[x_pref])\n\t"
- "pref 0, 96(%[x_pref])\n\t"
- "pref 0, 0(%[y_pref])\n\t"
- "pref 0, 32(%[y_pref])\n\t"
- "pref 0, 64(%[y_pref])\n\t"
- "pref 0, 96(%[y_pref])\n\t"
-
- : : [x_pref] "r" (x_pref), [y_pref] "r" (y_pref)
- );
-
- x_pref += 16;
- y_pref += 16;
-
- LD_DP8_INC(x, 2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
- LD_DP8_INC(y, 2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
+ FLOAT *x_pref, *y_pref;
+ BLASLONG pref_offset;
+
+ pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ x_pref = x + pref_offset + 32 + 8;
+
+ pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
+ if (pref_offset > 0)
+ {
+ pref_offset = L1_DATA_LINESIZE - pref_offset;
+ pref_offset = pref_offset / sizeof(FLOAT);
+ }
+ y_pref = y + pref_offset + 32 + 8;
+
+ LD_DP4_INC(x, 2, vx0, vx1, vx2, vx3);
+ LD_DP4_INC(y, 2, vy0, vy1, vy2, vy3);
PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i);
- PCKEVOD_D2_DP(vx3, vx2, vx1r, vx1i);
- PCKEVOD_D2_DP(vx5, vx4, vx2r, vx2i);
- PCKEVOD_D2_DP(vx7, vx6, vx3r, vx3i);
-
PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i);
- PCKEVOD_D2_DP(vy3, vy2, vy1r, vy1i);
- PCKEVOD_D2_DP(vy5, vy4, vy2r, vy2i);
- PCKEVOD_D2_DP(vy7, vy6, vy3r, vy3i);
+ for (i = (n >> 3) - 1; i--;)
+ {
+ PREF_OFFSET(x_pref, 0);
+ PREF_OFFSET(x_pref, 32);
+ PREF_OFFSET(x_pref, 64);
+ PREF_OFFSET(x_pref, 96);
+ PREF_OFFSET(y_pref, 0);
+ PREF_OFFSET(y_pref, 32);
+ PREF_OFFSET(y_pref, 64);
+ PREF_OFFSET(y_pref, 96);
+ x_pref += 16;
+ y_pref += 16;
+
+ vx4 = LD_DP(x); x += 2;
+ vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
+ dot0 += (vx0r * vy0r);
+ vx5 = LD_DP(x); x += 2;
+ vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_DP(y); y += 2;
+ vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
+ dot2 += (vx1r * vy1r);
+ vy5 = LD_DP(y); y += 2;
+ vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_DP(x); x += 2;
+ vx7 = LD_DP(x); x += 2;
+ vy6 = LD_DP(y); y += 2;
+ vy7 = LD_DP(y); y += 2;
+ vx8 = LD_DP(x); x += 2;
+ dot0 -= (vx0i * vy0i);
+ vx9 = LD_DP(x); x += 2;
+ vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
+ dot1 += (vx0r * vy0i);
+ vy8 = LD_DP(y); y += 2;
+ vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy9 = LD_DP(y); y += 2;
+ vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
+ dot3 += (vx1r * vy1i);
+ vx10 = LD_DP(x); x += 2;
+ vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
+ vx11 = LD_DP(x); x += 2;
+ vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
+ dot4 += (vx2r * vy2r);
+ vy10 = LD_DP(y); y += 2;
+ vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
+ dot5 += (vx2i * vy2r);
+ vy11 = LD_DP(y); y += 2;
+ vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
+ vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
+ dot6 += (vx3r * vy3r);
+ vx0r = (v2f64) __msa_pckev_d((v2i64) vx9, (v2i64) vx8);
+ dot7 += (vx3i * vy3r);
+ vx0i = (v2f64) __msa_pckod_d((v2i64) vx9, (v2i64) vx8);
+ vy0r = (v2f64) __msa_pckev_d((v2i64) vy9, (v2i64) vy8);
+ vx2 = vx10;
+ vy0i = (v2f64) __msa_pckod_d((v2i64) vy9, (v2i64) vy8);
+ vx3 = vx11;
+ dot4 -= (vx2i * vy2i);
+ vy2 = vy10;
+ dot5 += (vx2r * vy2i);
+ vy3 = vy11;
+ dot6 -= (vx3i * vy3i);
+ dot7 += (vx3r * vy3i);
+ }
+
+ vx4 = LD_DP(x); x += 2;
+ vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
dot0 += (vx0r * vy0r);
- dot0 OP1 (vx0i * vy0i);
- dot1 OP2 (vx0i * vy0r);
- dot1 += (vx0r * vy0i);
-
+ vx5 = LD_DP(x); x += 2;
+ vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_DP(y); y += 2;
+ vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r);
- dot2 OP1 (vx1i * vy1i);
- dot3 OP2 (vx1i * vy1r);
+ vy5 = LD_DP(y); y += 2;
+ vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_DP(x); x += 2;
+ vx7 = LD_DP(x); x += 2;
+ vy6 = LD_DP(y); y += 2;
+ vy7 = LD_DP(y); y += 2;
+ dot0 -= (vx0i * vy0i);
+ vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
+ dot1 += (vx0r * vy0i);
+ vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i);
-
+ vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
+ vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r);
- dot4 OP1 (vx2i * vy2i);
- dot5 OP2 (vx2i * vy2r);
- dot5 += (vx2r * vy2i);
-
+ vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
+ dot5 += (vx2i * vy2r);
+ vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
+ vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r);
- dot6 OP1 (vx3i * vy3i);
- dot7 OP2 (vx3i * vy3r);
+ dot7 += (vx3i * vy3r);
+ dot4 -= (vx2i * vy2i);
+ dot5 += (vx2r * vy2i);
+ dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}
}
- else
-#endif
- for (i = (n >> 3); i--;)
+ else if (n > 7)
{
- LD_DP8_INC(x, inc_x2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
- LD_DP8_INC(y, inc_y2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
+ LD_DP4_INC(x, inc_x2, vx0, vx1, vx2, vx3);
+ LD_DP4_INC(y, inc_y2, vy0, vy1, vy2, vy3);
PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i);
- PCKEVOD_D2_DP(vx3, vx2, vx1r, vx1i);
- PCKEVOD_D2_DP(vx5, vx4, vx2r, vx2i);
- PCKEVOD_D2_DP(vx7, vx6, vx3r, vx3i);
-
PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i);
- PCKEVOD_D2_DP(vy3, vy2, vy1r, vy1i);
- PCKEVOD_D2_DP(vy5, vy4, vy2r, vy2i);
- PCKEVOD_D2_DP(vy7, vy6, vy3r, vy3i);
- dot0 += (vx0r * vy0r);
- dot0 OP1 (vx0i * vy0i);
- dot1 OP2 (vx0i * vy0r);
- dot1 += (vx0r * vy0i);
+ for (i = (n >> 3) - 1; i--;)
+ {
+ vx4 = LD_DP(x); x += inc_x2;
+ vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
+ dot0 += (vx0r * vy0r);
+ vx5 = LD_DP(x); x += inc_x2;
+ vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_DP(y); y += inc_y2;
+ vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
+ dot2 += (vx1r * vy1r);
+ vy5 = LD_DP(y); y += inc_y2;
+ vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_DP(x); x += inc_x2;
+ vx7 = LD_DP(x); x += inc_x2;
+ vy6 = LD_DP(y); y += inc_y2;
+ vy7 = LD_DP(y); y += inc_y2;
+ vx8 = LD_DP(x); x += inc_x2;
+ dot0 -= (vx0i * vy0i);
+ vx9 = LD_DP(x); x += inc_x2;
+ vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
+ dot1 += (vx0r * vy0i);
+ vy8 = LD_DP(y); y += inc_y2;
+ vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy9 = LD_DP(y); y += inc_y2;
+ vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
+ dot3 += (vx1r * vy1i);
+ vx10 = LD_DP(x); x += inc_x2;
+ vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
+ vx11 = LD_DP(x); x += inc_x2;
+ vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
+ dot4 += (vx2r * vy2r);
+ vy10 = LD_DP(y); y += inc_y2;
+ vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
+ dot5 += (vx2i * vy2r);
+ vy11 = LD_DP(y); y += inc_y2;
+ vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
+ vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
+ dot6 += (vx3r * vy3r);
+ vx0r = (v2f64) __msa_pckev_d((v2i64) vx9, (v2i64) vx8);
+ dot7 += (vx3i * vy3r);
+ vx0i = (v2f64) __msa_pckod_d((v2i64) vx9, (v2i64) vx8);
+ vy0r = (v2f64) __msa_pckev_d((v2i64) vy9, (v2i64) vy8);
+ vx2 = vx10;
+ vy0i = (v2f64) __msa_pckod_d((v2i64) vy9, (v2i64) vy8);
+ vx3 = vx11;
+ dot4 -= (vx2i * vy2i);
+ vy2 = vy10;
+ dot5 += (vx2r * vy2i);
+ vy3 = vy11;
+ dot6 -= (vx3i * vy3i);
+ dot7 += (vx3r * vy3i);
+ }
+ vx4 = LD_DP(x); x += inc_x2;
+ vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
+ dot0 += (vx0r * vy0r);
+ vx5 = LD_DP(x); x += inc_x2;
+ vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
+ dot1 += (vx0i * vy0r);
+ vy4 = LD_DP(y); y += inc_y2;
+ vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r);
- dot2 OP1 (vx1i * vy1i);
- dot3 OP2 (vx1i * vy1r);
+ vy5 = LD_DP(y); y += inc_y2;
+ vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
+ dot3 += (vx1i * vy1r);
+ vx6 = LD_DP(x); x += inc_x2;
+ vx7 = LD_DP(x); x += inc_x2;
+ vy6 = LD_DP(y); y += inc_y2;
+ vy7 = LD_DP(y); y += inc_y2;
+ dot0 -= (vx0i * vy0i);
+ vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
+ dot1 += (vx0r * vy0i);
+ vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
+ dot2 -= (vx1i * vy1i);
+ vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i);
-
+ vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
+ vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r);
- dot4 OP1 (vx2i * vy2i);
- dot5 OP2 (vx2i * vy2r);
- dot5 += (vx2r * vy2i);
-
+ vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
+ dot5 += (vx2i * vy2r);
+ vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
+ vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r);
- dot6 OP1 (vx3i * vy3i);
- dot7 OP2 (vx3i * vy3r);
+ dot7 += (vx3i * vy3r);
+ dot4 -= (vx2i * vy2i);
+ dot5 += (vx2r * vy2i);
+ dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}
projects / platform / upstream / openblas.git / commitdiff