26 #ifndef EIGEN_COEFFBASED_PRODUCT_H
27 #define EIGEN_COEFFBASED_PRODUCT_H
46 template<
int Traversal,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
47 struct product_coeff_impl;
49 template<
int StorageOrder,
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
50 struct product_packet_impl;
52 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
53 struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> >
55 typedef MatrixXpr XprKind;
56 typedef typename remove_all<LhsNested>::type _LhsNested;
57 typedef typename remove_all<RhsNested>::type _RhsNested;
58 typedef typename scalar_product_traits<typename _LhsNested::Scalar, typename _RhsNested::Scalar>::ReturnType Scalar;
59 typedef typename promote_storage_type<typename traits<_LhsNested>::StorageKind,
61 typedef typename promote_index_type<typename traits<_LhsNested>::Index,
65 LhsCoeffReadCost = _LhsNested::CoeffReadCost,
66 RhsCoeffReadCost = _RhsNested::CoeffReadCost,
67 LhsFlags = _LhsNested::Flags,
68 RhsFlags = _RhsNested::Flags,
70 RowsAtCompileTime = _LhsNested::RowsAtCompileTime,
71 ColsAtCompileTime = _RhsNested::ColsAtCompileTime,
74 MaxRowsAtCompileTime = _LhsNested::MaxRowsAtCompileTime,
75 MaxColsAtCompileTime = _RhsNested::MaxColsAtCompileTime,
80 SameType = is_same<typename _LhsNested::Scalar,typename _RhsNested::Scalar>::value,
83 && (ColsAtCompileTime ==
Dynamic
84 || ( (ColsAtCompileTime % packet_traits<Scalar>::size) == 0
90 && (RowsAtCompileTime ==
Dynamic
91 || ( (RowsAtCompileTime % packet_traits<Scalar>::size) == 0
96 EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1
97 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0
98 : (RhsRowMajor && !CanVectorizeLhs),
101 | (EvalToRowMajor ? RowMajorBit : 0)
105 | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ?
PacketAccessBit : 0),
116 CanVectorizeInner = SameType
121 && (InnerSize % packet_traits<Scalar>::size == 0)
127 template<
typename LhsNested,
typename RhsNested,
int NestingFlags>
129 : internal::no_assignment_operator,
130 public MatrixBase<CoeffBasedProduct<LhsNested, RhsNested, NestingFlags> >
144 PacketSize = internal::packet_traits<Scalar>::size,
145 InnerSize = internal::traits<CoeffBasedProduct>::InnerSize,
147 CanVectorizeInner = internal::traits<CoeffBasedProduct>::CanVectorizeInner
151 Unroll ? InnerSize-1 :
Dynamic,
152 _LhsNested, _RhsNested, Scalar> ScalarCoeffImpl;
159 :
Base(), m_lhs(other.m_lhs), m_rhs(other.m_rhs)
162 template<
typename Lhs,
typename Rhs>
164 : m_lhs(lhs), m_rhs(rhs)
168 EIGEN_STATIC_ASSERT((internal::is_same<typename Lhs::RealScalar, typename Rhs::RealScalar>::value),
169 YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY)
171 &&
"invalid matrix product"
172 &&
"if you wanted a coeff-wise or a dot product use the respective explicit functions");
181 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
191 const Index row = RowsAtCompileTime == 1 ? 0 : index;
192 const Index col = RowsAtCompileTime == 1 ? index : 0;
193 ScalarCoeffImpl::run(row, col, m_lhs, m_rhs, res);
197 template<
int LoadMode>
202 Unroll ? InnerSize-1 :
Dynamic,
204 ::run(row, col, m_lhs, m_rhs, res);
215 const _LhsNested&
lhs()
const {
return m_lhs; }
216 const _RhsNested&
rhs()
const {
return m_rhs; }
221 template<
int DiagonalIndex>
229 typename internal::add_const_on_value_type<LhsNested>::type
m_lhs;
230 typename internal::add_const_on_value_type<RhsNested>::type
m_rhs;
239 template<
typename Lhs,
typename Rhs,
int N,
typename PlainObject>
240 struct nested<CoeffBasedProduct<Lhs,Rhs,EvalBeforeNestingBit|EvalBeforeAssigningBit>, N, PlainObject>
242 typedef PlainObject
const& type;
253 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
254 struct product_coeff_impl<
DefaultTraversal, UnrollingIndex, Lhs, Rhs, RetScalar>
256 typedef typename Lhs::Index Index;
259 product_coeff_impl<DefaultTraversal, UnrollingIndex-1, Lhs, Rhs, RetScalar>::run(row, col, lhs, rhs, res);
260 res += lhs.coeff(row, UnrollingIndex) * rhs.coeff(UnrollingIndex, col);
264 template<
typename Lhs,
typename Rhs,
typename RetScalar>
267 typedef typename Lhs::Index Index;
268 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
270 res = lhs.coeff(row, 0) * rhs.coeff(0, col);
274 template<
typename Lhs,
typename Rhs,
typename RetScalar>
277 typedef typename Lhs::Index Index;
278 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar& res)
280 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
281 res = lhs.coeff(row, 0) * rhs.coeff(0, col);
282 for(Index i = 1; i < lhs.cols(); ++i)
283 res += lhs.coeff(row, i) * rhs.coeff(i, col);
291 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet>
292 struct product_coeff_vectorized_unroller
294 typedef typename Lhs::Index Index;
295 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
296 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
298 product_coeff_vectorized_unroller<UnrollingIndex-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
299 pres =
padd(pres,
pmul( lhs.template packet<Aligned>(row, UnrollingIndex) , rhs.template packet<Aligned>(UnrollingIndex, col) ));
303 template<
typename Lhs,
typename Rhs,
typename Packet>
304 struct product_coeff_vectorized_unroller<0, Lhs, Rhs, Packet>
306 typedef typename Lhs::Index Index;
307 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::PacketScalar &pres)
309 pres =
pmul(lhs.template packet<Aligned>(row, 0) , rhs.template packet<Aligned>(0, col));
313 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename RetScalar>
316 typedef typename Lhs::PacketScalar Packet;
317 typedef typename Lhs::Index Index;
318 enum { PacketSize = packet_traits<typename Lhs::Scalar>::size };
319 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, RetScalar &res)
322 product_coeff_vectorized_unroller<UnrollingIndex+1-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, pres);
323 product_coeff_impl<DefaultTraversal,UnrollingIndex,Lhs,Rhs,RetScalar>::run(row, col, lhs, rhs, res);
328 template<
typename Lhs,
typename Rhs,
int LhsRows = Lhs::RowsAtCompileTime,
int RhsCols = Rhs::ColsAtCompileTime>
329 struct product_coeff_vectorized_dyn_selector
331 typedef typename Lhs::Index Index;
332 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
334 res = lhs.row(row).transpose().cwiseProduct(rhs.col(col)).sum();
340 template<
typename Lhs,
typename Rhs,
int RhsCols>
341 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,RhsCols>
343 typedef typename Lhs::Index Index;
344 static EIGEN_STRONG_INLINE void run(Index , Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
346 res = lhs.transpose().cwiseProduct(rhs.col(col)).sum();
350 template<
typename Lhs,
typename Rhs,
int LhsRows>
351 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,LhsRows,1>
353 typedef typename Lhs::Index Index;
354 static EIGEN_STRONG_INLINE void run(Index row, Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
356 res = lhs.row(row).transpose().cwiseProduct(rhs).sum();
360 template<
typename Lhs,
typename Rhs>
361 struct product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,1>
363 typedef typename Lhs::Index Index;
364 static EIGEN_STRONG_INLINE void run(Index , Index ,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
366 res = lhs.transpose().cwiseProduct(rhs).sum();
370 template<
typename Lhs,
typename Rhs,
typename RetScalar>
373 typedef typename Lhs::Index Index;
374 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs,
typename Lhs::Scalar &res)
376 product_coeff_vectorized_dyn_selector<Lhs,Rhs>::run(row, col, lhs, rhs, res);
384 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
385 struct product_packet_impl<
RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
387 typedef typename Lhs::Index Index;
388 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
390 product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
391 res =
pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res);
395 template<
int UnrollingIndex,
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
396 struct product_packet_impl<
ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode>
398 typedef typename Lhs::Index Index;
399 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
401 product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, res);
402 res =
pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res);
406 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
407 struct product_packet_impl<
RowMajor, 0, Lhs, Rhs, Packet, LoadMode>
409 typedef typename Lhs::Index Index;
410 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
412 res =
pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
416 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
417 struct product_packet_impl<
ColMajor, 0, Lhs, Rhs, Packet, LoadMode>
419 typedef typename Lhs::Index Index;
420 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet &res)
422 res =
pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
426 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
427 struct product_packet_impl<
RowMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
429 typedef typename Lhs::Index Index;
430 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
432 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
433 res =
pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col));
434 for(Index i = 1; i < lhs.cols(); ++i)
435 res =
pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res);
439 template<
typename Lhs,
typename Rhs,
typename Packet,
int LoadMode>
440 struct product_packet_impl<
ColMajor,
Dynamic, Lhs, Rhs, Packet, LoadMode>
442 typedef typename Lhs::Index Index;
443 static EIGEN_STRONG_INLINE void run(Index row, Index col,
const Lhs& lhs,
const Rhs& rhs, Packet& res)
445 eigen_assert(lhs.cols()>0 &&
"you are using a non initialized matrix");
446 res =
pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col)));
447 for(Index i = 1; i < lhs.cols(); ++i)
448 res =
pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res);
456 #endif // EIGEN_COEFFBASED_PRODUCT_H