28 const std::array<float32x4_t, 8>
exp_tab =
32 vdupq_n_f32(0.0416598916054f),
33 vdupq_n_f32(0.500000596046f),
34 vdupq_n_f32(0.0014122662833f),
35 vdupq_n_f32(1.00000011921f),
36 vdupq_n_f32(0.00833693705499f),
37 vdupq_n_f32(0.166665703058f),
38 vdupq_n_f32(0.000195780929062f),
43 const std::array<float32x4_t, 8>
log_tab =
46 vdupq_n_f32(-2.29561495781f),
47 vdupq_n_f32(-2.47071170807f),
48 vdupq_n_f32(-5.68692588806f),
49 vdupq_n_f32(-0.165253549814f),
50 vdupq_n_f32(5.17591238022f),
51 vdupq_n_f32(0.844007015228f),
52 vdupq_n_f32(4.58445882797f),
53 vdupq_n_f32(0.0141278216615f),
57 #ifndef DOXYGEN_SKIP_THIS 60 static const float32x4_t CONST_1 = vdupq_n_f32(1.f);
62 const int32x4_t z = vcvtq_s32_f32(val);
63 const float32x4_t r = vcvtq_f32_s32(z);
65 return vbslq_f32(vcgtq_f32(r, val), vsubq_f32(r, CONST_1), r);
70 float32x2_t sqrt_reciprocal = vrsqrte_f32(x);
71 sqrt_reciprocal = vmul_f32(vrsqrts_f32(vmul_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
72 sqrt_reciprocal = vmul_f32(vrsqrts_f32(vmul_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
74 return sqrt_reciprocal;
79 float32x4_t sqrt_reciprocal = vrsqrteq_f32(x);
80 sqrt_reciprocal = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
81 sqrt_reciprocal = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
83 return sqrt_reciprocal;
86 inline float32x2_t
vinv_f32(float32x2_t x)
88 float32x2_t recip = vrecpe_f32(x);
89 recip = vmul_f32(vrecps_f32(x, recip), recip);
90 recip = vmul_f32(vrecps_f32(x, recip), recip);
94 inline float32x4_t
vinvq_f32(float32x4_t x)
96 float32x4_t recip = vrecpeq_f32(x);
97 recip = vmulq_f32(vrecpsq_f32(x, recip), recip);
98 recip = vmulq_f32(vrecpsq_f32(x, recip), recip);
102 inline float32x4_t
vtaylor_polyq_f32(float32x4_t x,
const std::array<float32x4_t, 8> &coeffs)
104 float32x4_t
A = vmlaq_f32(coeffs[0], coeffs[4], x);
105 float32x4_t
B = vmlaq_f32(coeffs[2], coeffs[6], x);
106 float32x4_t C = vmlaq_f32(coeffs[1], coeffs[5], x);
107 float32x4_t D = vmlaq_f32(coeffs[3], coeffs[7], x);
108 float32x4_t x2 = vmulq_f32(x, x);
109 float32x4_t x4 = vmulq_f32(x2, x2);
110 float32x4_t res = vmlaq_f32(vmlaq_f32(A, B, x2), vmlaq_f32(C, D, x2), x4);
114 inline float32x4_t
vexpq_f32(float32x4_t x)
116 static const float32x4_t CONST_LN2 = vdupq_n_f32(0.6931471805f);
117 static const float32x4_t CONST_INV_LN2 = vdupq_n_f32(1.4426950408f);
118 static const float32x4_t CONST_0 = vdupq_n_f32(0.f);
119 static const int32x4_t CONST_NEGATIVE_126 = vdupq_n_s32(-126);
122 int32x4_t m = vcvtq_s32_f32(vmulq_f32(x, CONST_INV_LN2));
123 float32x4_t val = vmlsq_f32(x, vcvtq_f32_s32(m), CONST_LN2);
129 poly = vreinterpretq_f32_s32(vqaddq_s32(vreinterpretq_s32_f32(poly), vqshlq_n_s32(m, 23)));
130 poly = vbslq_f32(vcltq_s32(m, CONST_NEGATIVE_126), CONST_0, poly);
135 inline float32x4_t
vlogq_f32(float32x4_t x)
137 static const int32x4_t CONST_127 = vdupq_n_s32(127);
138 static const float32x4_t CONST_LN2 = vdupq_n_f32(0.6931471805f);
141 int32x4_t m = vsubq_s32(vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_f32(x), 23)), CONST_127);
142 float32x4_t val = vreinterpretq_f32_s32(vsubq_s32(vreinterpretq_s32_f32(x), vshlq_n_s32(m, 23)));
148 poly = vmlaq_f32(poly, vcvtq_f32_s32(m), CONST_LN2);
153 inline float32x4_t
vtanhq_f32(float32x4_t val)
155 static const float32x4_t CONST_1 = vdupq_n_f32(1.f);
156 static const float32x4_t CONST_2 = vdupq_n_f32(2.f);
157 static const float32x4_t CONST_MIN_TANH = vdupq_n_f32(-10.f);
158 static const float32x4_t CONST_MAX_TANH = vdupq_n_f32(10.f);
160 float32x4_t x = vminq_f32(vmaxq_f32(val, CONST_MIN_TANH), CONST_MAX_TANH);
161 float32x4_t exp2x =
vexpq_f32(vmulq_f32(CONST_2, x));
162 float32x4_t num = vsubq_f32(exp2x, CONST_1);
163 float32x4_t den = vaddq_f32(exp2x, CONST_1);
168 inline float32x4_t
vpowq_f32(float32x4_t val, float32x4_t n)
174 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC 176 const std::array<float16x8_t, 8> exp_tab_f16 =
180 vdupq_n_f16(0.0416598916054f),
181 vdupq_n_f16(0.500000596046f),
182 vdupq_n_f16(0.0014122662833f),
183 vdupq_n_f16(1.00000011921f),
184 vdupq_n_f16(0.00833693705499f),
185 vdupq_n_f16(0.166665703058f),
186 vdupq_n_f16(0.000195780929062f),
191 const std::array<float16x8_t, 8> log_tab_f16 =
194 vdupq_n_f16(-2.29561495781f),
195 vdupq_n_f16(-2.47071170807f),
196 vdupq_n_f16(-5.68692588806f),
197 vdupq_n_f16(-0.165253549814f),
198 vdupq_n_f16(5.17591238022f),
199 vdupq_n_f16(0.844007015228f),
200 vdupq_n_f16(4.58445882797f),
201 vdupq_n_f16(0.0141278216615f),
205 #ifndef DOXYGEN_SKIP_THIS 206 inline float16x4_t vinvsqrt_f16(float16x4_t x)
208 float16x4_t sqrt_reciprocal = vrsqrte_f16(x);
209 sqrt_reciprocal = vmul_f16(vrsqrts_f16(vmul_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
210 sqrt_reciprocal = vmul_f16(vrsqrts_f16(vmul_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
211 return sqrt_reciprocal;
214 inline float16x8_t vinvsqrtq_f16(float16x8_t x)
216 float16x8_t sqrt_reciprocal = vrsqrteq_f16(x);
217 sqrt_reciprocal = vmulq_f16(vrsqrtsq_f16(vmulq_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
218 sqrt_reciprocal = vmulq_f16(vrsqrtsq_f16(vmulq_f16(x, sqrt_reciprocal), sqrt_reciprocal), sqrt_reciprocal);
219 return sqrt_reciprocal;
222 inline float16x4_t vinv_f16(float16x4_t x)
224 float16x4_t recip = vrecpe_f16(x);
225 recip = vmul_f16(vrecps_f16(x, recip), recip);
226 recip = vmul_f16(vrecps_f16(x, recip), recip);
230 inline float16x8_t vinvq_f16(float16x8_t x)
232 float16x8_t recip = vrecpeq_f16(x);
233 recip = vmulq_f16(vrecpsq_f16(x, recip), recip);
234 recip = vmulq_f16(vrecpsq_f16(x, recip), recip);
238 inline float16x8_t vtanhq_f16(float16x8_t val)
240 const float16x8_t CONST_1 = vdupq_n_f16(1.f);
241 const float16x8_t CONST_2 = vdupq_n_f16(2.f);
242 const float16x8_t CONST_MIN_TANH = vdupq_n_f16(-10.f);
243 const float16x8_t CONST_MAX_TANH = vdupq_n_f16(10.f);
245 const float16x8_t x = vminq_f16(vmaxq_f16(val, CONST_MIN_TANH), CONST_MAX_TANH);
246 const float16x8_t exp2x = vexpq_f16(vmulq_f16(CONST_2, x));
247 const float16x8_t num = vsubq_f16(exp2x, CONST_1);
248 const float16x8_t den = vaddq_f16(exp2x, CONST_1);
249 const float16x8_t
tanh = vmulq_f16(num, vinvq_f16(den));
253 inline float16x8_t vtaylor_polyq_f16(float16x8_t x,
const std::array<float16x8_t, 8> &coeffs)
255 const float16x8_t
A = vaddq_f16(coeffs[0], vmulq_f16(coeffs[4], x));
256 const float16x8_t
B = vaddq_f16(coeffs[2], vmulq_f16(coeffs[6], x));
257 const float16x8_t C = vaddq_f16(coeffs[1], vmulq_f16(coeffs[5], x));
258 const float16x8_t D = vaddq_f16(coeffs[3], vmulq_f16(coeffs[7], x));
259 const float16x8_t x2 = vmulq_f16(x, x);
260 const float16x8_t x4 = vmulq_f16(x2, x2);
261 const float16x8_t res = vaddq_f16(vaddq_f16(A, vmulq_f16(B, x2)), vmulq_f16(vaddq_f16(C, vmulq_f16(D, x2)), x4));
265 inline float16x8_t vexpq_f16(float16x8_t x)
267 static const float16x8_t CONST_LN2 = vdupq_n_f16(0.6931471805f);
268 static const float16x8_t CONST_INV_LN2 = vdupq_n_f16(1.4426950408f);
269 static const float16x8_t CONST_0 = vdupq_n_f16(0.f);
270 static const int16x8_t CONST_NEGATIVE_126 = vdupq_n_s16(-126);
273 const int16x8_t m = vcvtq_s16_f16(vmulq_f16(x, CONST_INV_LN2));
274 const float16x8_t val = vsubq_f16(x, vmulq_f16(vcvtq_f16_s16(m), CONST_LN2));
277 float16x8_t poly = vtaylor_polyq_f16(val, exp_tab_f16);
280 poly = vreinterpretq_f16_s16(vqaddq_s16(vreinterpretq_s16_f16(poly), vqshlq_n_s16(m, 9)));
281 poly = vbslq_f16(vcltq_s16(m, CONST_NEGATIVE_126), CONST_0, poly);
286 inline float16x8_t vlogq_f16(float16x8_t x)
288 static const int16x8_t CONST_127 = vdupq_n_s16(127);
289 static const float16x8_t CONST_LN2 = vdupq_n_f16(0.6931471805f);
292 const int16x8_t m = vsubq_s16(vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_f16(x), 9)), CONST_127);
293 const float16x8_t val = vreinterpretq_f16_s16(vsubq_s16(vreinterpretq_s16_f16(x), vshlq_n_s16(m, 9)));
296 float16x8_t poly = vtaylor_polyq_f16(val, log_tab_f16);
299 poly = vaddq_f16(poly, vmulq_f16(vcvtq_f16_s16(m), CONST_LN2));
304 inline float16x8_t vpowq_f16(float16x8_t val, float16x8_t n)
306 return vexpq_f16(vmulq_f16(n, vlogq_f16(val)));
float32x4_t vtanhq_f32(float32x4_t val)
Calculate hyperbolic tangent.
float32x4_t vinvsqrtq_f32(float32x4_t x)
Calculate inverse square root.
float32x2_t vinv_f32(float32x2_t x)
Calculate reciprocal.
float32x4_t vtaylor_polyq_f32(float32x4_t x, const std::array< float32x4_t, 8 > &coeffs)
Perform a 7th degree polynomial approximation using Estrin's method.
This file contains all available output stages for GEMMLowp on OpenCL.
float32x4_t vfloorq_f32(float32x4_t val)
Calculate floor of a vector.
const std::array< float32x4_t, 8 > exp_tab
Exponent polynomial coefficients.
float32x4_t vpowq_f32(float32x4_t val, float32x4_t n)
Calculate n power of a number.
float32x4_t vlogq_f32(float32x4_t x)
Calculate logarithm.
float32x2_t vinvsqrt_f32(float32x2_t x)
Calculate inverse square root.
float32x4_t vexpq_f32(float32x4_t x)
Calculate exponential.
float32x4_t vinvq_f32(float32x4_t x)
Calculate reciprocal.
fixed_point< T > tanh(fixed_point< T > x)
const std::array< float32x4_t, 8 > log_tab
Logarithm polynomial coefficients.