v_uint16x8 v_0 = v_setall_u16(0);
for ( ; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 _v_mask = v_load(mask + x);
- v_uint16x8 v_mask, dummy;
- v_expand(_v_mask, v_mask, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
v_uint16x8 v_src = v_load(src + x);
v_src = v_src & v_mask;
v_uint16x8 v_0 = v_setall_u16(0);
for ( ; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 _v_mask = v_load(mask + x);
- v_uint16x8 v_mask, dummy;
- v_expand(_v_mask, v_mask, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
v_uint16x8 v_src0, v_src1, v_src2;
v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2);
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
+ v_uint16x8 v_masku16 = v_load_expand(mask + x);
v_uint32x4 v_masku320, v_masku321;
v_expand(v_masku16, v_masku320, v_masku321);
v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_masku320 == v_reinterpret_as_u32(v_0)));
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
+ v_uint16x8 v_masku16 = v_load_expand(mask + x);
v_uint32x4 v_masku320, v_masku321;
v_expand(v_masku16, v_masku320, v_masku321);
v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_masku320 == v_reinterpret_as_u32(v_0)));
{
for ( ; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 v_mask, dummy;
- v_expand(v_load(mask + x), v_mask, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
v_uint16x8 v_src = v_load(src + x);
v_src = v_src & v_mask;
{
for ( ; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 v_mask, dummy;
- v_expand(v_load(mask + x), v_mask, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
v_uint16x8 v_src0, v_src1, v_src2;
v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2);
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
- v_uint32x4 v_masku32, dummy1;
- v_expand(v_masku16, v_masku32, dummy1);
+ v_uint32x4 v_masku32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
v_expand(v_masku32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
- v_uint32x4 v_masku32, dummy1;
- v_expand(v_masku16, v_masku32, dummy1);
+ v_uint32x4 v_masku32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
v_expand(v_masku32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
- v_uint32x4 v_masku32, dummy1;
- v_expand(v_masku16, v_masku32, dummy1);
+ v_uint32x4 v_masku32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
v_expand(v_masku32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 v_masku16, dummy0;
- v_expand(v_load(mask + x), v_masku16, dummy0);
- v_uint32x4 v_masku32, dummy1;
- v_expand(v_masku16, v_masku32, dummy1);
+ v_uint32x4 v_masku32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
v_expand(v_masku32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
+ v_uint16x8 v_mask16 = v_load_expand(mask + x);
v_uint32x4 v_mask0, v_mask1;
- v_expand(stub, v_mask0, v_mask1);
+ v_expand(v_mask16, v_mask0, v_mask1);
v_mask0 = ~(v_mask0 == v_0);
v_mask1 = ~(v_mask1 == v_0);
v_uint16x8 v_src = v_load(src + x);
{
for ( ; x <= len - cVectorWidth ; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
+ v_uint16x8 v_mask16 = v_load_expand(mask + x);
v_uint32x4 v_mask0, v_mask1;
- v_expand(stub, v_mask0, v_mask1);
+ v_expand(v_mask16, v_mask0, v_mask1);
v_mask0 = ~(v_mask0 == v_0);
v_mask1 = ~(v_mask1 == v_0);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 v_stub = v_load_expand(mask + x);
- v_uint32x4 v_stub0, v_stub1;
- v_expand(v_stub, v_stub0, v_stub1);
- v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_stub0 == v_0));
- v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_stub1 == v_0));
+ v_uint16x8 v_mask16 = v_load_expand(mask + x);
+ v_uint32x4 v_mask_0, v_mask_1;
+ v_expand(v_mask16, v_mask_0, v_mask_1);
+ v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_mask_0 == v_0));
+ v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_mask_1 == v_0));
v_float32x4 v_src0 = v_load(src + x);
v_float32x4 v_src1 = v_load(src + x + 4);
v_src0 = v_src0 & v_mask0;
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 v_stub = v_load_expand(mask + x);
- v_uint32x4 v_stub0, v_stub1;
- v_expand(v_stub, v_stub0, v_stub1);
- v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_stub0 == v_0));
- v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_stub1 == v_0));
+ v_uint16x8 v_mask16 = v_load_expand(mask + x);
+ v_uint32x4 v_mask_0, v_mask_1;
+ v_expand(v_mask16, v_mask_0, v_mask_1);
+ v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_mask_0 == v_0));
+ v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_mask_1 == v_0));
v_float32x4 v_src00, v_src10, v_src20, v_src01, v_src11, v_src21;
v_load_deinterleave(src + x * cn, v_src00, v_src10, v_src20);
int size = len * cn;
for (; x <= size - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 v_src = v_load(src + x);
- v_uint16x8 v_int, dummy;
- v_expand(v_src, v_int, dummy);
+ v_uint16x8 v_int = v_load_expand(src + x);
v_uint32x4 v_int0, v_int1;
v_expand(v_int, v_int0, v_int1);
}
else
{
- v_uint8x16 v_0 = v_setzero_u8();
+ v_uint16x8 v_0 = v_setzero_u16();
if (cn == 1)
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 v_mask = v_load(mask + x);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
- v_uint8x16 v_src = v_load(src + x);
- v_src = v_src & v_mask;
- v_uint16x8 v_int, dummy;
- v_expand(v_src, v_int, dummy);
+ v_uint16x8 v_src = v_load_expand(src + x);
+ v_uint16x8 v_int = v_src & v_mask;
v_uint32x4 v_int0, v_int1;
v_expand(v_int, v_int0, v_int1);
}
else if (cn == 3)
{
- for (; x <= len - cVectorWidth; x += cVectorWidth)
+ for (; x <= len - /*cVectorWidth*/16; x += cVectorWidth)
{
- v_uint8x16 v_mask = v_load(mask + x);
- v_mask = ~(v_mask == v_0);
v_uint8x16 v_src0, v_src1, v_src2;
v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2);
- v_src0 = v_src0 & v_mask;
- v_src1 = v_src1 & v_mask;
- v_src2 = v_src2 & v_mask;
v_uint16x8 v_int0, v_int1, v_int2, dummy;
v_expand(v_src0, v_int0, dummy);
v_expand(v_src1, v_int1, dummy);
v_expand(v_src2, v_int2, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
+ v_mask = ~(v_mask == v_0);
+ v_int0 = v_int0 & v_mask;
+ v_int1 = v_int1 & v_mask;
+ v_int2 = v_int2 & v_mask;
v_uint32x4 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21;
v_expand(v_int0, v_int00, v_int01);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 v_mask, dummy;
- v_expand(stub, v_mask, dummy);
+ v_uint32x4 v_mask = v_load_expand_q(mask + x);;
v_mask = ~(v_mask == v_0);
v_float32x4 v_src = v_load(src + x);
v_src = v_src & v_reinterpret_as_f32(v_mask);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 v_mask, dummy;
- v_expand(stub, v_mask, dummy);
+ v_uint32x4 v_mask = v_load_expand_q(mask + x);
v_mask = ~(v_mask == v_0);
v_float32x4 v_src0, v_src1, v_src2;
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
+ v_uint32x4 v_mask32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
- v_expand(stub0, v_masku640, v_masku641);
+ v_expand(v_mask32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
v_float64x2 v_mask1 = v_reinterpret_as_f64(~(v_masku641 == v_0));
v_float64x2 v_src0 = v_load(src + x);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
+ v_uint32x4 v_mask32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
- v_expand(stub0, v_masku640, v_masku641);
+ v_expand(v_mask32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
v_float64x2 v_mask1 = v_reinterpret_as_f64(~(v_masku641 == v_0));
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
- v_float32x4 v_mask0 = v_reinterpret_as_f32(~(stub0 == v_0));
- v_float32x4 v_mask1 = v_reinterpret_as_f32(~(stub1 == v_0));
+ v_uint32x4 v_mask32_0 = v_load_expand_q(mask + x);
+ v_uint32x4 v_mask32_1 = v_load_expand_q(mask + x + 4);
+ v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_mask32_0 == v_0));
+ v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_mask32_1 == v_0));
v_store(dst + x, v_load(dst + x) + ((v_load(src1 + x) * v_load(src2 + x)) & v_mask0));
v_store(dst + x + 4, v_load(dst + x + 4) + ((v_load(src1 + x + 4) * v_load(src2 + x + 4)) & v_mask1));
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
- v_float32x4 v_mask0 = v_reinterpret_as_f32(~(stub0 == v_0));
- v_float32x4 v_mask1 = v_reinterpret_as_f32(~(stub1 == v_0));
+ v_uint32x4 v_mask32_0 = v_load_expand_q(mask + x);
+ v_uint32x4 v_mask32_1 = v_load_expand_q(mask + x + 4);
+ v_float32x4 v_mask0 = v_reinterpret_as_f32(~(v_mask32_0 == v_0));
+ v_float32x4 v_mask1 = v_reinterpret_as_f32(~(v_mask32_1 == v_0));
v_float32x4 v_1src00, v_1src01, v_1src10, v_1src11, v_1src20, v_1src21;
v_float32x4 v_2src00, v_2src01, v_2src10, v_2src11, v_2src20, v_2src21;
int size = len * cn;
for (; x <= size - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 v_1src = v_load(src1 + x);
- v_uint8x16 v_2src = v_load(src2 + x);
-
- v_uint16x8 v_1int, v_2int, dummy;
- v_expand(v_1src, v_1int, dummy);
- v_expand(v_2src, v_2int, dummy);
+ v_uint16x8 v_1int = v_load_expand(src1 + x);
+ v_uint16x8 v_2int = v_load_expand(src2 + x);
v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1;
v_expand(v_1int, v_1int_0, v_1int_1);
}
else
{
- v_uint8x16 v_0 = v_setzero_u8();
+ v_uint16x8 v_0 = v_setzero_u16();
if (cn == 1)
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 v_mask = v_load(mask + x);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
v_mask = ~(v_mask == v_0);
- v_uint8x16 v_1src = v_load(src1 + x) & v_mask;
- v_uint8x16 v_2src = v_load(src2 + x) & v_mask;
-
- v_uint16x8 v_1int, v_2int, dummy;
- v_expand(v_1src, v_1int, dummy);
- v_expand(v_2src, v_2int, dummy);
+ v_uint16x8 v_1int = v_load_expand(src1 + x) & v_mask;
+ v_uint16x8 v_2int = v_load_expand(src2 + x) & v_mask;
v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1;
v_expand(v_1int, v_1int_0, v_1int_1);
}
else if (cn == 3)
{
- for (; x <= len - cVectorWidth; x += cVectorWidth)
+ for (; x <= len - /*cVectorWidth*/16; x += cVectorWidth)
{
- v_uint8x16 v_mask = v_load(mask + x);
- v_mask = ~(v_mask == v_0);
v_uint8x16 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2;
v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2);
v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2);
- v_1src0 = v_1src0 & v_mask;
- v_1src1 = v_1src1 & v_mask;
- v_1src2 = v_1src2 & v_mask;
- v_2src0 = v_2src0 & v_mask;
- v_2src1 = v_2src1 & v_mask;
- v_2src2 = v_2src2 & v_mask;
v_uint16x8 v_1int0, v_1int1, v_1int2, v_2int0, v_2int1, v_2int2, dummy;
v_expand(v_1src0, v_1int0, dummy);
v_expand(v_2src1, v_2int1, dummy);
v_expand(v_2src2, v_2int2, dummy);
+ v_uint16x8 v_mask = v_load_expand(mask + x);
+ v_mask = ~(v_mask == v_0);
+ v_1int0 = v_1int0 & v_mask;
+ v_1int1 = v_1int1 & v_mask;
+ v_1int2 = v_1int2 & v_mask;
+ v_2int0 = v_2int0 & v_mask;
+ v_2int1 = v_2int1 & v_mask;
+ v_2int2 = v_2int2 & v_mask;
+
v_uint32x4 v_1int00, v_1int01, v_1int10, v_1int11, v_1int20, v_1int21;
v_uint32x4 v_2int00, v_2int01, v_2int10, v_2int11, v_2int20, v_2int21;
v_expand(v_1int0, v_1int00, v_1int01);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 v_mask, dummy;
- v_expand(stub, v_mask, dummy);
+ v_uint32x4 v_mask = v_load_expand_q(mask + x);
v_mask = ~(v_mask == v_0);
v_float32x4 v_1src = v_load(src1 + x);
v_float32x4 v_2src = v_load(src2 + x);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 v_mask, dummy;
- v_expand(stub, v_mask, dummy);
+ v_uint32x4 v_mask = v_load_expand_q(mask + x);
v_mask = ~(v_mask == v_0);
v_float32x4 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2;
v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2);
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
+ v_uint32x4 v_mask32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
- v_expand(stub0, v_masku640, v_masku641);
+ v_expand(v_mask32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
v_float64x2 v_mask1 = v_reinterpret_as_f64(~(v_masku641 == v_0));
{
for (; x <= len - cVectorWidth; x += cVectorWidth)
{
- v_uint16x8 stub = v_load_expand(mask + x);
- v_uint32x4 stub0, stub1;
- v_expand(stub, stub0, stub1);
+ v_uint32x4 v_mask32 = v_load_expand_q(mask + x);
v_uint64x2 v_masku640, v_masku641;
- v_expand(stub0, v_masku640, v_masku641);
+ v_expand(v_mask32, v_masku640, v_masku641);
v_float64x2 v_mask0 = v_reinterpret_as_f64(~(v_masku640 == v_0));
v_float64x2 v_mask1 = v_reinterpret_as_f64(~(v_masku641 == v_0));
int size = len * cn;
for (; x <= size - cVectorWidth; x += cVectorWidth)
{
- v_uint8x16 v_src = v_load(src + x);
- v_uint16x8 v_int, dummy;
- v_expand(v_src, v_int, dummy);
+ v_uint16x8 v_src16 = v_load_expand(src + x);
v_uint32x4 v_int_0, v_int_1;
- v_expand(v_int, v_int_0, v_int_1);
+ v_expand(v_src16, v_int_0, v_int_1);
v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0);
v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1);