1 // This file is part of Eigen, a lightweight C++ template library 2 // for linear algebra. 3 // 4 // Copyright (C) 2009 Mark Borgerding mark a borgerding net 5 // 6 // This Source Code Form is subject to the terms of the Mozilla 7 // Public License v. 2.0. If a copy of the MPL was not distributed 8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 10 namespace Eigen { 11 12 namespace internal { 13 14 // This FFT implementation was derived from kissfft http:sourceforge.net/projects/kissfft 15 // Copyright 2003-2009 Mark Borgerding 16 17 template <typename _Scalar> 18 struct kiss_cpx_fft 19 { 20 typedef _Scalar Scalar; 21 typedef std::complex<Scalar> Complex; 22 std::vector<Complex> m_twiddles; 23 std::vector<int> m_stageRadix; 24 std::vector<int> m_stageRemainder; 25 std::vector<Complex> m_scratchBuf; 26 bool m_inverse; 27 28 inline make_twiddleskiss_cpx_fft29 void make_twiddles(int nfft,bool inverse) 30 { 31 using std::acos; 32 m_inverse = inverse; 33 m_twiddles.resize(nfft); 34 Scalar phinc = (inverse?2:-2)* acos( (Scalar) -1) / nfft; 35 for (int i=0;i<nfft;++i) 36 m_twiddles[i] = exp( Complex(0,i*phinc) ); 37 } 38 factorizekiss_cpx_fft39 void factorize(int nfft) 40 { 41 //start factoring out 4's, then 2's, then 3,5,7,9,... 42 int n= nfft; 43 int p=4; 44 do { 45 while (n % p) { 46 switch (p) { 47 case 4: p = 2; break; 48 case 2: p = 3; break; 49 default: p += 2; break; 50 } 51 if (p*p>n) 52 p=n;// impossible to have a factor > sqrt(n) 53 } 54 n /= p; 55 m_stageRadix.push_back(p); 56 m_stageRemainder.push_back(n); 57 if ( p > 5 ) 58 m_scratchBuf.resize(p); // scratchbuf will be needed in bfly_generic 59 }while(n>1); 60 } 61 62 template <typename _Src> 63 inline workkiss_cpx_fft64 void work( int stage,Complex * xout, const _Src * xin, size_t fstride,size_t in_stride) 65 { 66 int p = m_stageRadix[stage]; 67 int m = m_stageRemainder[stage]; 68 Complex * Fout_beg = xout; 69 Complex * Fout_end = xout + p*m; 70 71 if (m>1) { 72 do{ 73 // recursive call: 74 // DFT of size m*p performed by doing 75 // p instances of smaller DFTs of size m, 76 // each one takes a decimated version of the input 77 work(stage+1, xout , xin, fstride*p,in_stride); 78 xin += fstride*in_stride; 79 }while( (xout += m) != Fout_end ); 80 }else{ 81 do{ 82 *xout = *xin; 83 xin += fstride*in_stride; 84 }while(++xout != Fout_end ); 85 } 86 xout=Fout_beg; 87 88 // recombine the p smaller DFTs 89 switch (p) { 90 case 2: bfly2(xout,fstride,m); break; 91 case 3: bfly3(xout,fstride,m); break; 92 case 4: bfly4(xout,fstride,m); break; 93 case 5: bfly5(xout,fstride,m); break; 94 default: bfly_generic(xout,fstride,m,p); break; 95 } 96 } 97 98 inline bfly2kiss_cpx_fft99 void bfly2( Complex * Fout, const size_t fstride, int m) 100 { 101 for (int k=0;k<m;++k) { 102 Complex t = Fout[m+k] * m_twiddles[k*fstride]; 103 Fout[m+k] = Fout[k] - t; 104 Fout[k] += t; 105 } 106 } 107 108 inline bfly4kiss_cpx_fft109 void bfly4( Complex * Fout, const size_t fstride, const size_t m) 110 { 111 Complex scratch[6]; 112 int negative_if_inverse = m_inverse * -2 +1; 113 for (size_t k=0;k<m;++k) { 114 scratch[0] = Fout[k+m] * m_twiddles[k*fstride]; 115 scratch[1] = Fout[k+2*m] * m_twiddles[k*fstride*2]; 116 scratch[2] = Fout[k+3*m] * m_twiddles[k*fstride*3]; 117 scratch[5] = Fout[k] - scratch[1]; 118 119 Fout[k] += scratch[1]; 120 scratch[3] = scratch[0] + scratch[2]; 121 scratch[4] = scratch[0] - scratch[2]; 122 scratch[4] = Complex( scratch[4].imag()*negative_if_inverse , -scratch[4].real()* negative_if_inverse ); 123 124 Fout[k+2*m] = Fout[k] - scratch[3]; 125 Fout[k] += scratch[3]; 126 Fout[k+m] = scratch[5] + scratch[4]; 127 Fout[k+3*m] = scratch[5] - scratch[4]; 128 } 129 } 130 131 inline bfly3kiss_cpx_fft132 void bfly3( Complex * Fout, const size_t fstride, const size_t m) 133 { 134 size_t k=m; 135 const size_t m2 = 2*m; 136 Complex *tw1,*tw2; 137 Complex scratch[5]; 138 Complex epi3; 139 epi3 = m_twiddles[fstride*m]; 140 141 tw1=tw2=&m_twiddles[0]; 142 143 do{ 144 scratch[1]=Fout[m] * *tw1; 145 scratch[2]=Fout[m2] * *tw2; 146 147 scratch[3]=scratch[1]+scratch[2]; 148 scratch[0]=scratch[1]-scratch[2]; 149 tw1 += fstride; 150 tw2 += fstride*2; 151 Fout[m] = Complex( Fout->real() - Scalar(.5)*scratch[3].real() , Fout->imag() - Scalar(.5)*scratch[3].imag() ); 152 scratch[0] *= epi3.imag(); 153 *Fout += scratch[3]; 154 Fout[m2] = Complex( Fout[m].real() + scratch[0].imag() , Fout[m].imag() - scratch[0].real() ); 155 Fout[m] += Complex( -scratch[0].imag(),scratch[0].real() ); 156 ++Fout; 157 }while(--k); 158 } 159 160 inline bfly5kiss_cpx_fft161 void bfly5( Complex * Fout, const size_t fstride, const size_t m) 162 { 163 Complex *Fout0,*Fout1,*Fout2,*Fout3,*Fout4; 164 size_t u; 165 Complex scratch[13]; 166 Complex * twiddles = &m_twiddles[0]; 167 Complex *tw; 168 Complex ya,yb; 169 ya = twiddles[fstride*m]; 170 yb = twiddles[fstride*2*m]; 171 172 Fout0=Fout; 173 Fout1=Fout0+m; 174 Fout2=Fout0+2*m; 175 Fout3=Fout0+3*m; 176 Fout4=Fout0+4*m; 177 178 tw=twiddles; 179 for ( u=0; u<m; ++u ) { 180 scratch[0] = *Fout0; 181 182 scratch[1] = *Fout1 * tw[u*fstride]; 183 scratch[2] = *Fout2 * tw[2*u*fstride]; 184 scratch[3] = *Fout3 * tw[3*u*fstride]; 185 scratch[4] = *Fout4 * tw[4*u*fstride]; 186 187 scratch[7] = scratch[1] + scratch[4]; 188 scratch[10] = scratch[1] - scratch[4]; 189 scratch[8] = scratch[2] + scratch[3]; 190 scratch[9] = scratch[2] - scratch[3]; 191 192 *Fout0 += scratch[7]; 193 *Fout0 += scratch[8]; 194 195 scratch[5] = scratch[0] + Complex( 196 (scratch[7].real()*ya.real() ) + (scratch[8].real() *yb.real() ), 197 (scratch[7].imag()*ya.real()) + (scratch[8].imag()*yb.real()) 198 ); 199 200 scratch[6] = Complex( 201 (scratch[10].imag()*ya.imag()) + (scratch[9].imag()*yb.imag()), 202 -(scratch[10].real()*ya.imag()) - (scratch[9].real()*yb.imag()) 203 ); 204 205 *Fout1 = scratch[5] - scratch[6]; 206 *Fout4 = scratch[5] + scratch[6]; 207 208 scratch[11] = scratch[0] + 209 Complex( 210 (scratch[7].real()*yb.real()) + (scratch[8].real()*ya.real()), 211 (scratch[7].imag()*yb.real()) + (scratch[8].imag()*ya.real()) 212 ); 213 214 scratch[12] = Complex( 215 -(scratch[10].imag()*yb.imag()) + (scratch[9].imag()*ya.imag()), 216 (scratch[10].real()*yb.imag()) - (scratch[9].real()*ya.imag()) 217 ); 218 219 *Fout2=scratch[11]+scratch[12]; 220 *Fout3=scratch[11]-scratch[12]; 221 222 ++Fout0;++Fout1;++Fout2;++Fout3;++Fout4; 223 } 224 } 225 226 /* perform the butterfly for one stage of a mixed radix FFT */ 227 inline bfly_generickiss_cpx_fft228 void bfly_generic( 229 Complex * Fout, 230 const size_t fstride, 231 int m, 232 int p 233 ) 234 { 235 int u,k,q1,q; 236 Complex * twiddles = &m_twiddles[0]; 237 Complex t; 238 int Norig = static_cast<int>(m_twiddles.size()); 239 Complex * scratchbuf = &m_scratchBuf[0]; 240 241 for ( u=0; u<m; ++u ) { 242 k=u; 243 for ( q1=0 ; q1<p ; ++q1 ) { 244 scratchbuf[q1] = Fout[ k ]; 245 k += m; 246 } 247 248 k=u; 249 for ( q1=0 ; q1<p ; ++q1 ) { 250 int twidx=0; 251 Fout[ k ] = scratchbuf[0]; 252 for (q=1;q<p;++q ) { 253 twidx += static_cast<int>(fstride) * k; 254 if (twidx>=Norig) twidx-=Norig; 255 t=scratchbuf[q] * twiddles[twidx]; 256 Fout[ k ] += t; 257 } 258 k += m; 259 } 260 } 261 } 262 }; 263 264 template <typename _Scalar> 265 struct kissfft_impl 266 { 267 typedef _Scalar Scalar; 268 typedef std::complex<Scalar> Complex; 269 clearkissfft_impl270 void clear() 271 { 272 m_plans.clear(); 273 m_realTwiddles.clear(); 274 } 275 276 inline fwdkissfft_impl277 void fwd( Complex * dst,const Complex *src,int nfft) 278 { 279 get_plan(nfft,false).work(0, dst, src, 1,1); 280 } 281 282 inline fwd2kissfft_impl283 void fwd2( Complex * dst,const Complex *src,int n0,int n1) 284 { 285 EIGEN_UNUSED_VARIABLE(dst); 286 EIGEN_UNUSED_VARIABLE(src); 287 EIGEN_UNUSED_VARIABLE(n0); 288 EIGEN_UNUSED_VARIABLE(n1); 289 } 290 291 inline inv2kissfft_impl292 void inv2( Complex * dst,const Complex *src,int n0,int n1) 293 { 294 EIGEN_UNUSED_VARIABLE(dst); 295 EIGEN_UNUSED_VARIABLE(src); 296 EIGEN_UNUSED_VARIABLE(n0); 297 EIGEN_UNUSED_VARIABLE(n1); 298 } 299 300 // real-to-complex forward FFT 301 // perform two FFTs of src even and src odd 302 // then twiddle to recombine them into the half-spectrum format 303 // then fill in the conjugate symmetric half 304 inline fwdkissfft_impl305 void fwd( Complex * dst,const Scalar * src,int nfft) 306 { 307 if ( nfft&3 ) { 308 // use generic mode for odd 309 m_tmpBuf1.resize(nfft); 310 get_plan(nfft,false).work(0, &m_tmpBuf1[0], src, 1,1); 311 std::copy(m_tmpBuf1.begin(),m_tmpBuf1.begin()+(nfft>>1)+1,dst ); 312 }else{ 313 int ncfft = nfft>>1; 314 int ncfft2 = nfft>>2; 315 Complex * rtw = real_twiddles(ncfft2); 316 317 // use optimized mode for even real 318 fwd( dst, reinterpret_cast<const Complex*> (src), ncfft); 319 Complex dc = dst[0].real() + dst[0].imag(); 320 Complex nyquist = dst[0].real() - dst[0].imag(); 321 int k; 322 for ( k=1;k <= ncfft2 ; ++k ) { 323 Complex fpk = dst[k]; 324 Complex fpnk = conj(dst[ncfft-k]); 325 Complex f1k = fpk + fpnk; 326 Complex f2k = fpk - fpnk; 327 Complex tw= f2k * rtw[k-1]; 328 dst[k] = (f1k + tw) * Scalar(.5); 329 dst[ncfft-k] = conj(f1k -tw)*Scalar(.5); 330 } 331 dst[0] = dc; 332 dst[ncfft] = nyquist; 333 } 334 } 335 336 // inverse complex-to-complex 337 inline invkissfft_impl338 void inv(Complex * dst,const Complex *src,int nfft) 339 { 340 get_plan(nfft,true).work(0, dst, src, 1,1); 341 } 342 343 // half-complex to scalar 344 inline invkissfft_impl345 void inv( Scalar * dst,const Complex * src,int nfft) 346 { 347 if (nfft&3) { 348 m_tmpBuf1.resize(nfft); 349 m_tmpBuf2.resize(nfft); 350 std::copy(src,src+(nfft>>1)+1,m_tmpBuf1.begin() ); 351 for (int k=1;k<(nfft>>1)+1;++k) 352 m_tmpBuf1[nfft-k] = conj(m_tmpBuf1[k]); 353 inv(&m_tmpBuf2[0],&m_tmpBuf1[0],nfft); 354 for (int k=0;k<nfft;++k) 355 dst[k] = m_tmpBuf2[k].real(); 356 }else{ 357 // optimized version for multiple of 4 358 int ncfft = nfft>>1; 359 int ncfft2 = nfft>>2; 360 Complex * rtw = real_twiddles(ncfft2); 361 m_tmpBuf1.resize(ncfft); 362 m_tmpBuf1[0] = Complex( src[0].real() + src[ncfft].real(), src[0].real() - src[ncfft].real() ); 363 for (int k = 1; k <= ncfft / 2; ++k) { 364 Complex fk = src[k]; 365 Complex fnkc = conj(src[ncfft-k]); 366 Complex fek = fk + fnkc; 367 Complex tmp = fk - fnkc; 368 Complex fok = tmp * conj(rtw[k-1]); 369 m_tmpBuf1[k] = fek + fok; 370 m_tmpBuf1[ncfft-k] = conj(fek - fok); 371 } 372 get_plan(ncfft,true).work(0, reinterpret_cast<Complex*>(dst), &m_tmpBuf1[0], 1,1); 373 } 374 } 375 376 protected: 377 typedef kiss_cpx_fft<Scalar> PlanData; 378 typedef std::map<int,PlanData> PlanMap; 379 380 PlanMap m_plans; 381 std::map<int, std::vector<Complex> > m_realTwiddles; 382 std::vector<Complex> m_tmpBuf1; 383 std::vector<Complex> m_tmpBuf2; 384 385 inline PlanKeykissfft_impl386 int PlanKey(int nfft, bool isinverse) const { return (nfft<<1) | int(isinverse); } 387 388 inline get_plankissfft_impl389 PlanData & get_plan(int nfft, bool inverse) 390 { 391 // TODO look for PlanKey(nfft, ! inverse) and conjugate the twiddles 392 PlanData & pd = m_plans[ PlanKey(nfft,inverse) ]; 393 if ( pd.m_twiddles.size() == 0 ) { 394 pd.make_twiddles(nfft,inverse); 395 pd.factorize(nfft); 396 } 397 return pd; 398 } 399 400 inline real_twiddleskissfft_impl401 Complex * real_twiddles(int ncfft2) 402 { 403 using std::acos; 404 std::vector<Complex> & twidref = m_realTwiddles[ncfft2];// creates new if not there 405 if ( (int)twidref.size() != ncfft2 ) { 406 twidref.resize(ncfft2); 407 int ncfft= ncfft2<<1; 408 Scalar pi = acos( Scalar(-1) ); 409 for (int k=1;k<=ncfft2;++k) 410 twidref[k-1] = exp( Complex(0,-pi * (Scalar(k) / ncfft + Scalar(.5)) ) ); 411 } 412 return &twidref[0]; 413 } 414 }; 415 416 } // end namespace internal 417 418 } // end namespace Eigen 419 420 /* vim: set filetype=cpp et sw=2 ts=2 ai: */ 421