12 #ifndef EIGEN_PACKET_MATH_NEON_H
13 #define EIGEN_PACKET_MATH_NEON_H
19 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
25 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8
29 typedef float32x4_t Packet4f;
30 typedef int32x4_t Packet4i;
31 typedef uint32x4_t Packet4ui;
33 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
34 const Packet4f p4f_##NAME = pset1<Packet4f>(X)
36 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
37 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int>(X))
39 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
40 const Packet4i p4i_##NAME = pset1<Packet4i>(X)
42 #if defined(__llvm__) && !defined(__clang__)
44 #define EIGEN_INIT_NEON_PACKET2(X, Y) {{X, Y}}
45 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {{X, Y, Z, W}}
48 #define EIGEN_INIT_NEON_PACKET2(X, Y) {X, Y}
49 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {X, Y, Z, W}
53 #define __pld(x) asm volatile ( " pld [%[addr]]\n" :: [addr] "r" (x) : "cc" );
56 template<>
struct packet_traits<float> : default_packet_traits
58 typedef Packet4f type;
73 template<>
struct packet_traits<int> : default_packet_traits
75 typedef Packet4i type;
84 #if EIGEN_GNUC_AT_MOST(4,4) && !defined(__llvm__)
86 EIGEN_STRONG_INLINE float32x4_t vld1q_f32(
const float* x) { return ::vld1q_f32((
const float32_t*)x); }
87 EIGEN_STRONG_INLINE float32x2_t vld1_f32 (
const float* x) { return ::vld1_f32 ((
const float32_t*)x); }
88 EIGEN_STRONG_INLINE
void vst1q_f32(
float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
89 EIGEN_STRONG_INLINE
void vst1_f32 (
float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
92 template<>
struct unpacket_traits<Packet4f> {
typedef float type;
enum {size=4}; };
93 template<>
struct unpacket_traits<Packet4i> {
typedef int type;
enum {size=4}; };
95 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
return vdupq_n_f32(from); }
96 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
return vdupq_n_s32(from); }
98 template<> EIGEN_STRONG_INLINE Packet4f plset<float>(
const float& a)
100 Packet4f countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
101 return vaddq_f32(pset1<Packet4f>(a), countdown);
103 template<> EIGEN_STRONG_INLINE Packet4i plset<int>(
const int& a)
105 Packet4i countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
106 return vaddq_s32(pset1<Packet4i>(a), countdown);
109 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vaddq_f32(a,b); }
110 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vaddq_s32(a,b); }
112 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vsubq_f32(a,b); }
113 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vsubq_s32(a,b); }
115 template<> EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
return vnegq_f32(a); }
116 template<> EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
return vnegq_s32(a); }
118 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmulq_f32(a,b); }
119 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmulq_s32(a,b); }
121 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b)
123 Packet4f inv, restep, div;
130 inv = vrecpeq_f32(b);
134 restep = vrecpsq_f32(b, inv);
135 inv = vmulq_f32(restep, inv);
138 div = vmulq_f32(a, inv);
142 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& )
143 { eigen_assert(
false &&
"packet integer division are not supported by NEON");
144 return pset1<Packet4i>(0);
148 template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
return vmlaq_f32(c,a,b); }
149 template<> EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
return vmlaq_s32(c,a,b); }
151 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vminq_f32(a,b); }
152 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vminq_s32(a,b); }
154 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmaxq_f32(a,b); }
155 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmaxq_s32(a,b); }
158 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b)
160 return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
162 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vandq_s32(a,b); }
164 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b)
166 return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
168 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vorrq_s32(a,b); }
170 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b)
172 return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
174 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return veorq_s32(a,b); }
176 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b)
178 return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
180 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vbicq_s32(a,b); }
182 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f32(from); }
183 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from) { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s32(from); }
185 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f32(from); }
186 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s32(from); }
188 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from)
191 lo = vdup_n_f32(*from);
192 hi = vdup_n_f32(*(from+1));
193 return vcombine_f32(lo, hi);
195 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from)
198 lo = vdup_n_s32(*from);
199 hi = vdup_n_s32(*(from+1));
200 return vcombine_s32(lo, hi);
203 template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); }
204 template<> EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); }
206 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); }
207 template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); }
209 template<> EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) { __pld(addr); }
210 template<> EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) { __pld(addr); }
213 template<> EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a);
return x[0]; }
214 template<> EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
int EIGEN_ALIGN16 x[4]; vst1q_s32(x, a);
return x[0]; }
216 template<> EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
217 float32x2_t a_lo, a_hi;
220 a_r64 = vrev64q_f32(a);
221 a_lo = vget_low_f32(a_r64);
222 a_hi = vget_high_f32(a_r64);
223 return vcombine_f32(a_hi, a_lo);
225 template<> EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
226 int32x2_t a_lo, a_hi;
229 a_r64 = vrev64q_s32(a);
230 a_lo = vget_low_s32(a_r64);
231 a_hi = vget_high_s32(a_r64);
232 return vcombine_s32(a_hi, a_lo);
234 template<> EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
return vabsq_f32(a); }
235 template<> EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
return vabsq_s32(a); }
237 template<> EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a)
239 float32x2_t a_lo, a_hi, sum;
242 a_lo = vget_low_f32(a);
243 a_hi = vget_high_f32(a);
244 sum = vpadd_f32(a_lo, a_hi);
245 sum = vpadd_f32(sum, sum);
251 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(
const Packet4f* vecs)
253 float32x4x2_t vtrn1, vtrn2, res1, res2;
254 Packet4f sum1, sum2, sum;
258 vtrn1 = vzipq_f32(vecs[0], vecs[2]);
259 vtrn2 = vzipq_f32(vecs[1], vecs[3]);
260 res1 = vzipq_f32(vtrn1.val[0], vtrn2.val[0]);
261 res2 = vzipq_f32(vtrn1.val[1], vtrn2.val[1]);
264 sum1 = vaddq_f32(res1.val[0], res1.val[1]);
265 sum2 = vaddq_f32(res2.val[0], res2.val[1]);
266 sum = vaddq_f32(sum1, sum2);
271 template<> EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a)
273 int32x2_t a_lo, a_hi, sum;
276 a_lo = vget_low_s32(a);
277 a_hi = vget_high_s32(a);
278 sum = vpadd_s32(a_lo, a_hi);
279 sum = vpadd_s32(sum, sum);
285 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(
const Packet4i* vecs)
287 int32x4x2_t vtrn1, vtrn2, res1, res2;
288 Packet4i sum1, sum2, sum;
292 vtrn1 = vzipq_s32(vecs[0], vecs[2]);
293 vtrn2 = vzipq_s32(vecs[1], vecs[3]);
294 res1 = vzipq_s32(vtrn1.val[0], vtrn2.val[0]);
295 res2 = vzipq_s32(vtrn1.val[1], vtrn2.val[1]);
298 sum1 = vaddq_s32(res1.val[0], res1.val[1]);
299 sum2 = vaddq_s32(res2.val[0], res2.val[1]);
300 sum = vaddq_s32(sum1, sum2);
307 template<> EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a)
309 float32x2_t a_lo, a_hi, prod;
313 a_lo = vget_low_f32(a);
314 a_hi = vget_high_f32(a);
316 prod = vmul_f32(a_lo, a_hi);
318 prod = vmul_f32(prod, vrev64_f32(prod));
323 template<> EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a)
325 int32x2_t a_lo, a_hi, prod;
329 a_lo = vget_low_s32(a);
330 a_hi = vget_high_s32(a);
332 prod = vmul_s32(a_lo, a_hi);
334 prod = vmul_s32(prod, vrev64_s32(prod));
341 template<> EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a)
343 float32x2_t a_lo, a_hi, min;
346 a_lo = vget_low_f32(a);
347 a_hi = vget_high_f32(a);
348 min = vpmin_f32(a_lo, a_hi);
349 min = vpmin_f32(min, min);
354 template<> EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a)
356 int32x2_t a_lo, a_hi, min;
359 a_lo = vget_low_s32(a);
360 a_hi = vget_high_s32(a);
361 min = vpmin_s32(a_lo, a_hi);
362 min = vpmin_s32(min, min);
369 template<> EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a)
371 float32x2_t a_lo, a_hi, max;
374 a_lo = vget_low_f32(a);
375 a_hi = vget_high_f32(a);
376 max = vpmax_f32(a_lo, a_hi);
377 max = vpmax_f32(max, max);
382 template<> EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a)
384 int32x2_t a_lo, a_hi, max;
387 a_lo = vget_low_s32(a);
388 a_hi = vget_high_s32(a);
389 max = vpmax_s32(a_lo, a_hi);
390 max = vpmax_s32(max, max);
398 #define PALIGN_NEON(Offset,Type,Command) \
400 struct palign_impl<Offset,Type>\
402 EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\
405 first = Command(first, second, Offset);\
409 PALIGN_NEON(0,Packet4f,vextq_f32)
410 PALIGN_NEON(1,Packet4f,vextq_f32)
411 PALIGN_NEON(2,Packet4f,vextq_f32)
412 PALIGN_NEON(3,Packet4f,vextq_f32)
413 PALIGN_NEON(0,Packet4i,vextq_s32)
414 PALIGN_NEON(1,Packet4i,vextq_s32)
415 PALIGN_NEON(2,Packet4i,vextq_s32)
416 PALIGN_NEON(3,Packet4i,vextq_s32)
424 #endif // EIGEN_PACKET_MATH_NEON_H