ei_kissfft_impl.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2009 Mark Borgerding mark a borgerding net
5 //
6 // Eigen is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU Lesser General Public
8 // License as published by the Free Software Foundation; either
9 // version 3 of the License, or (at your option) any later version.
10 //
11 // Alternatively, you can redistribute it and/or
12 // modify it under the terms of the GNU General Public License as
13 // published by the Free Software Foundation; either version 2 of
14 // the License, or (at your option) any later version.
15 //
16 // Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
17 // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 // FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
19 // GNU General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License and a copy of the GNU General Public License along with
23 // Eigen. If not, see <http://www.gnu.org/licenses/>.
24 
25 namespace Eigen {
26 
27 namespace internal {
28 
29  // This FFT implementation was derived from kissfft http:sourceforge.net/projects/kissfft
30  // Copyright 2003-2009 Mark Borgerding
31 
32 template <typename _Scalar>
33 struct kiss_cpx_fft
34 {
35  typedef _Scalar Scalar;
36  typedef std::complex<Scalar> Complex;
37  std::vector<Complex> m_twiddles;
38  std::vector<int> m_stageRadix;
39  std::vector<int> m_stageRemainder;
40  std::vector<Complex> m_scratchBuf;
41  bool m_inverse;
42 
43  inline
44  void make_twiddles(int nfft,bool inverse)
45  {
46  m_inverse = inverse;
47  m_twiddles.resize(nfft);
48  Scalar phinc = (inverse?2:-2)* acos( (Scalar) -1) / nfft;
49  for (int i=0;i<nfft;++i)
50  m_twiddles[i] = exp( Complex(0,i*phinc) );
51  }
52 
53  void factorize(int nfft)
54  {
55  //start factoring out 4's, then 2's, then 3,5,7,9,...
56  int n= nfft;
57  int p=4;
58  do {
59  while (n % p) {
60  switch (p) {
61  case 4: p = 2; break;
62  case 2: p = 3; break;
63  default: p += 2; break;
64  }
65  if (p*p>n)
66  p=n;// impossible to have a factor > sqrt(n)
67  }
68  n /= p;
69  m_stageRadix.push_back(p);
70  m_stageRemainder.push_back(n);
71  if ( p > 5 )
72  m_scratchBuf.resize(p); // scratchbuf will be needed in bfly_generic
73  }while(n>1);
74  }
75 
76  template <typename _Src>
77  inline
78  void work( int stage,Complex * xout, const _Src * xin, size_t fstride,size_t in_stride)
79  {
80  int p = m_stageRadix[stage];
81  int m = m_stageRemainder[stage];
82  Complex * Fout_beg = xout;
83  Complex * Fout_end = xout + p*m;
84 
85  if (m>1) {
86  do{
87  // recursive call:
88  // DFT of size m*p performed by doing
89  // p instances of smaller DFTs of size m,
90  // each one takes a decimated version of the input
91  work(stage+1, xout , xin, fstride*p,in_stride);
92  xin += fstride*in_stride;
93  }while( (xout += m) != Fout_end );
94  }else{
95  do{
96  *xout = *xin;
97  xin += fstride*in_stride;
98  }while(++xout != Fout_end );
99  }
100  xout=Fout_beg;
101 
102  // recombine the p smaller DFTs
103  switch (p) {
104  case 2: bfly2(xout,fstride,m); break;
105  case 3: bfly3(xout,fstride,m); break;
106  case 4: bfly4(xout,fstride,m); break;
107  case 5: bfly5(xout,fstride,m); break;
108  default: bfly_generic(xout,fstride,m,p); break;
109  }
110  }
111 
112  inline
113  void bfly2( Complex * Fout, const size_t fstride, int m)
114  {
115  for (int k=0;k<m;++k) {
116  Complex t = Fout[m+k] * m_twiddles[k*fstride];
117  Fout[m+k] = Fout[k] - t;
118  Fout[k] += t;
119  }
120  }
121 
122  inline
123  void bfly4( Complex * Fout, const size_t fstride, const size_t m)
124  {
125  Complex scratch[6];
126  int negative_if_inverse = m_inverse * -2 +1;
127  for (size_t k=0;k<m;++k) {
128  scratch[0] = Fout[k+m] * m_twiddles[k*fstride];
129  scratch[1] = Fout[k+2*m] * m_twiddles[k*fstride*2];
130  scratch[2] = Fout[k+3*m] * m_twiddles[k*fstride*3];
131  scratch[5] = Fout[k] - scratch[1];
132 
133  Fout[k] += scratch[1];
134  scratch[3] = scratch[0] + scratch[2];
135  scratch[4] = scratch[0] - scratch[2];
136  scratch[4] = Complex( scratch[4].imag()*negative_if_inverse , -scratch[4].real()* negative_if_inverse );
137 
138  Fout[k+2*m] = Fout[k] - scratch[3];
139  Fout[k] += scratch[3];
140  Fout[k+m] = scratch[5] + scratch[4];
141  Fout[k+3*m] = scratch[5] - scratch[4];
142  }
143  }
144 
145  inline
146  void bfly3( Complex * Fout, const size_t fstride, const size_t m)
147  {
148  size_t k=m;
149  const size_t m2 = 2*m;
150  Complex *tw1,*tw2;
151  Complex scratch[5];
152  Complex epi3;
153  epi3 = m_twiddles[fstride*m];
154 
155  tw1=tw2=&m_twiddles[0];
156 
157  do{
158  scratch[1]=Fout[m] * *tw1;
159  scratch[2]=Fout[m2] * *tw2;
160 
161  scratch[3]=scratch[1]+scratch[2];
162  scratch[0]=scratch[1]-scratch[2];
163  tw1 += fstride;
164  tw2 += fstride*2;
165  Fout[m] = Complex( Fout->real() - Scalar(.5)*scratch[3].real() , Fout->imag() - Scalar(.5)*scratch[3].imag() );
166  scratch[0] *= epi3.imag();
167  *Fout += scratch[3];
168  Fout[m2] = Complex( Fout[m].real() + scratch[0].imag() , Fout[m].imag() - scratch[0].real() );
169  Fout[m] += Complex( -scratch[0].imag(),scratch[0].real() );
170  ++Fout;
171  }while(--k);
172  }
173 
174  inline
175  void bfly5( Complex * Fout, const size_t fstride, const size_t m)
176  {
177  Complex *Fout0,*Fout1,*Fout2,*Fout3,*Fout4;
178  size_t u;
179  Complex scratch[13];
180  Complex * twiddles = &m_twiddles[0];
181  Complex *tw;
182  Complex ya,yb;
183  ya = twiddles[fstride*m];
184  yb = twiddles[fstride*2*m];
185 
186  Fout0=Fout;
187  Fout1=Fout0+m;
188  Fout2=Fout0+2*m;
189  Fout3=Fout0+3*m;
190  Fout4=Fout0+4*m;
191 
192  tw=twiddles;
193  for ( u=0; u<m; ++u ) {
194  scratch[0] = *Fout0;
195 
196  scratch[1] = *Fout1 * tw[u*fstride];
197  scratch[2] = *Fout2 * tw[2*u*fstride];
198  scratch[3] = *Fout3 * tw[3*u*fstride];
199  scratch[4] = *Fout4 * tw[4*u*fstride];
200 
201  scratch[7] = scratch[1] + scratch[4];
202  scratch[10] = scratch[1] - scratch[4];
203  scratch[8] = scratch[2] + scratch[3];
204  scratch[9] = scratch[2] - scratch[3];
205 
206  *Fout0 += scratch[7];
207  *Fout0 += scratch[8];
208 
209  scratch[5] = scratch[0] + Complex(
210  (scratch[7].real()*ya.real() ) + (scratch[8].real() *yb.real() ),
211  (scratch[7].imag()*ya.real()) + (scratch[8].imag()*yb.real())
212  );
213 
214  scratch[6] = Complex(
215  (scratch[10].imag()*ya.imag()) + (scratch[9].imag()*yb.imag()),
216  -(scratch[10].real()*ya.imag()) - (scratch[9].real()*yb.imag())
217  );
218 
219  *Fout1 = scratch[5] - scratch[6];
220  *Fout4 = scratch[5] + scratch[6];
221 
222  scratch[11] = scratch[0] +
223  Complex(
224  (scratch[7].real()*yb.real()) + (scratch[8].real()*ya.real()),
225  (scratch[7].imag()*yb.real()) + (scratch[8].imag()*ya.real())
226  );
227 
228  scratch[12] = Complex(
229  -(scratch[10].imag()*yb.imag()) + (scratch[9].imag()*ya.imag()),
230  (scratch[10].real()*yb.imag()) - (scratch[9].real()*ya.imag())
231  );
232 
233  *Fout2=scratch[11]+scratch[12];
234  *Fout3=scratch[11]-scratch[12];
235 
236  ++Fout0;++Fout1;++Fout2;++Fout3;++Fout4;
237  }
238  }
239 
240  /* perform the butterfly for one stage of a mixed radix FFT */
241  inline
242  void bfly_generic(
243  Complex * Fout,
244  const size_t fstride,
245  int m,
246  int p
247  )
248  {
249  int u,k,q1,q;
250  Complex * twiddles = &m_twiddles[0];
251  Complex t;
252  int Norig = static_cast<int>(m_twiddles.size());
253  Complex * scratchbuf = &m_scratchBuf[0];
254 
255  for ( u=0; u<m; ++u ) {
256  k=u;
257  for ( q1=0 ; q1<p ; ++q1 ) {
258  scratchbuf[q1] = Fout[ k ];
259  k += m;
260  }
261 
262  k=u;
263  for ( q1=0 ; q1<p ; ++q1 ) {
264  int twidx=0;
265  Fout[ k ] = scratchbuf[0];
266  for (q=1;q<p;++q ) {
267  twidx += static_cast<int>(fstride) * k;
268  if (twidx>=Norig) twidx-=Norig;
269  t=scratchbuf[q] * twiddles[twidx];
270  Fout[ k ] += t;
271  }
272  k += m;
273  }
274  }
275  }
276 };
277 
278 template <typename _Scalar>
279 struct kissfft_impl
280 {
281  typedef _Scalar Scalar;
282  typedef std::complex<Scalar> Complex;
283 
284  void clear()
285  {
286  m_plans.clear();
287  m_realTwiddles.clear();
288  }
289 
290  inline
291  void fwd( Complex * dst,const Complex *src,int nfft)
292  {
293  get_plan(nfft,false).work(0, dst, src, 1,1);
294  }
295 
296  inline
297  void fwd2( Complex * dst,const Complex *src,int n0,int n1)
298  {
299  EIGEN_UNUSED_VARIABLE(dst);
300  EIGEN_UNUSED_VARIABLE(src);
301  EIGEN_UNUSED_VARIABLE(n0);
302  EIGEN_UNUSED_VARIABLE(n1);
303  }
304 
305  inline
306  void inv2( Complex * dst,const Complex *src,int n0,int n1)
307  {
308  EIGEN_UNUSED_VARIABLE(dst);
309  EIGEN_UNUSED_VARIABLE(src);
310  EIGEN_UNUSED_VARIABLE(n0);
311  EIGEN_UNUSED_VARIABLE(n1);
312  }
313 
314  // real-to-complex forward FFT
315  // perform two FFTs of src even and src odd
316  // then twiddle to recombine them into the half-spectrum format
317  // then fill in the conjugate symmetric half
318  inline
319  void fwd( Complex * dst,const Scalar * src,int nfft)
320  {
321  if ( nfft&3 ) {
322  // use generic mode for odd
323  m_tmpBuf1.resize(nfft);
324  get_plan(nfft,false).work(0, &m_tmpBuf1[0], src, 1,1);
325  std::copy(m_tmpBuf1.begin(),m_tmpBuf1.begin()+(nfft>>1)+1,dst );
326  }else{
327  int ncfft = nfft>>1;
328  int ncfft2 = nfft>>2;
329  Complex * rtw = real_twiddles(ncfft2);
330 
331  // use optimized mode for even real
332  fwd( dst, reinterpret_cast<const Complex*> (src), ncfft);
333  Complex dc = dst[0].real() + dst[0].imag();
334  Complex nyquist = dst[0].real() - dst[0].imag();
335  int k;
336  for ( k=1;k <= ncfft2 ; ++k ) {
337  Complex fpk = dst[k];
338  Complex fpnk = conj(dst[ncfft-k]);
339  Complex f1k = fpk + fpnk;
340  Complex f2k = fpk - fpnk;
341  Complex tw= f2k * rtw[k-1];
342  dst[k] = (f1k + tw) * Scalar(.5);
343  dst[ncfft-k] = conj(f1k -tw)*Scalar(.5);
344  }
345  dst[0] = dc;
346  dst[ncfft] = nyquist;
347  }
348  }
349 
350  // inverse complex-to-complex
351  inline
352  void inv(Complex * dst,const Complex *src,int nfft)
353  {
354  get_plan(nfft,true).work(0, dst, src, 1,1);
355  }
356 
357  // half-complex to scalar
358  inline
359  void inv( Scalar * dst,const Complex * src,int nfft)
360  {
361  if (nfft&3) {
362  m_tmpBuf1.resize(nfft);
363  m_tmpBuf2.resize(nfft);
364  std::copy(src,src+(nfft>>1)+1,m_tmpBuf1.begin() );
365  for (int k=1;k<(nfft>>1)+1;++k)
366  m_tmpBuf1[nfft-k] = conj(m_tmpBuf1[k]);
367  inv(&m_tmpBuf2[0],&m_tmpBuf1[0],nfft);
368  for (int k=0;k<nfft;++k)
369  dst[k] = m_tmpBuf2[k].real();
370  }else{
371  // optimized version for multiple of 4
372  int ncfft = nfft>>1;
373  int ncfft2 = nfft>>2;
374  Complex * rtw = real_twiddles(ncfft2);
375  m_tmpBuf1.resize(ncfft);
376  m_tmpBuf1[0] = Complex( src[0].real() + src[ncfft].real(), src[0].real() - src[ncfft].real() );
377  for (int k = 1; k <= ncfft / 2; ++k) {
378  Complex fk = src[k];
379  Complex fnkc = conj(src[ncfft-k]);
380  Complex fek = fk + fnkc;
381  Complex tmp = fk - fnkc;
382  Complex fok = tmp * conj(rtw[k-1]);
383  m_tmpBuf1[k] = fek + fok;
384  m_tmpBuf1[ncfft-k] = conj(fek - fok);
385  }
386  get_plan(ncfft,true).work(0, reinterpret_cast<Complex*>(dst), &m_tmpBuf1[0], 1,1);
387  }
388  }
389 
390  protected:
391  typedef kiss_cpx_fft<Scalar> PlanData;
392  typedef std::map<int,PlanData> PlanMap;
393 
394  PlanMap m_plans;
395  std::map<int, std::vector<Complex> > m_realTwiddles;
396  std::vector<Complex> m_tmpBuf1;
397  std::vector<Complex> m_tmpBuf2;
398 
399  inline
400  int PlanKey(int nfft, bool isinverse) const { return (nfft<<1) | int(isinverse); }
401 
402  inline
403  PlanData & get_plan(int nfft, bool inverse)
404  {
405  // TODO look for PlanKey(nfft, ! inverse) and conjugate the twiddles
406  PlanData & pd = m_plans[ PlanKey(nfft,inverse) ];
407  if ( pd.m_twiddles.size() == 0 ) {
408  pd.make_twiddles(nfft,inverse);
409  pd.factorize(nfft);
410  }
411  return pd;
412  }
413 
414  inline
415  Complex * real_twiddles(int ncfft2)
416  {
417  std::vector<Complex> & twidref = m_realTwiddles[ncfft2];// creates new if not there
418  if ( (int)twidref.size() != ncfft2 ) {
419  twidref.resize(ncfft2);
420  int ncfft= ncfft2<<1;
421  Scalar pi = acos( Scalar(-1) );
422  for (int k=1;k<=ncfft2;++k)
423  twidref[k-1] = exp( Complex(0,-pi * (Scalar(k) / ncfft + Scalar(.5)) ) );
424  }
425  return &twidref[0];
426  }
427 };
428 
429 } // end namespace internal
430 
431 } // end namespace Eigen
432 
433 /* vim: set filetype=cpp et sw=2 ts=2 ai: */