1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 // Class for generating spectrogram slices from a waveform. 17 // Initialize() should be called before calls to other functions. Once 18 // Initialize() has been called and returned true, The Compute*() functions can 19 // be called repeatedly with sequential input data (ie. the first element of the 20 // next input vector directly follows the last element of the previous input 21 // vector). Whenever enough audio samples are buffered to produce a 22 // new frame, it will be placed in output. Output is cleared on each 23 // call to Compute*(). This class is thread-unsafe, and should only be 24 // called from one thread at a time. 25 // With the default parameters, the output of this class should be very 26 // close to the results of the following MATLAB code: 27 // overlap_samples = window_length_samples - step_samples; 28 // window = hann(window_length_samples, 'periodic'); 29 // S = abs(spectrogram(audio, window, overlap_samples)).^2; 30 31 #ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_SPECTROGRAM_H_ 32 #define TENSORFLOW_LITE_KERNELS_INTERNAL_SPECTROGRAM_H_ 33 34 #include <complex> 35 #include <deque> 36 #include <vector> 37 38 #include "third_party/fft2d/fft.h" 39 40 namespace tflite { 41 namespace internal { 42 43 class Spectrogram { 44 public: Spectrogram()45 Spectrogram() : initialized_(false) {} ~Spectrogram()46 ~Spectrogram() {} 47 48 // Initializes the class with a given window length and step length 49 // (both in samples). Internally a Hann window is used as the window 50 // function. Returns true on success, after which calls to Process() 51 // are possible. window_length must be greater than 1 and step 52 // length must be greater than 0. 53 bool Initialize(int window_length, int step_length); 54 55 // Initialize with an explicit window instead of a length. 56 bool Initialize(const std::vector<double>& window, int step_length); 57 58 // Processes an arbitrary amount of audio data (contained in input) 59 // to yield complex spectrogram frames. After a successful call to 60 // Initialize(), Process() may be called repeatedly with new input data 61 // each time. The audio input is buffered internally, and the output 62 // vector is populated with as many temporally-ordered spectral slices 63 // as it is possible to generate from the input. The output is cleared 64 // on each call before the new frames (if any) are added. 65 // 66 // The template parameters can be float or double. 67 template <class InputSample, class OutputSample> 68 bool ComputeComplexSpectrogram( 69 const std::vector<InputSample>& input, 70 std::vector<std::vector<std::complex<OutputSample>>>* output); 71 72 // This function works as the one above, but returns the power 73 // (the L2 norm, or the squared magnitude) of each complex value. 74 template <class InputSample, class OutputSample> 75 bool ComputeSquaredMagnitudeSpectrogram( 76 const std::vector<InputSample>& input, 77 std::vector<std::vector<OutputSample>>* output); 78 79 // Return reference to the window function used internally. GetWindow()80 const std::vector<double>& GetWindow() const { return window_; } 81 82 // Return the number of frequency channels in the spectrogram. output_frequency_channels()83 int output_frequency_channels() const { return output_frequency_channels_; } 84 85 private: 86 template <class InputSample> 87 bool GetNextWindowOfSamples(const std::vector<InputSample>& input, 88 int* input_start); 89 void ProcessCoreFFT(); 90 91 int fft_length_; 92 int output_frequency_channels_; 93 int window_length_; 94 int step_length_; 95 bool initialized_; 96 int samples_to_next_step_; 97 98 std::vector<double> window_; 99 std::vector<double> fft_input_output_; 100 std::deque<double> input_queue_; 101 102 // Working data areas for the FFT routines. 103 std::vector<int> fft_integer_working_area_; 104 std::vector<double> fft_double_working_area_; 105 }; 106 107 // Explicit instantiations in spectrogram.cc. 108 109 extern template bool Spectrogram::ComputeComplexSpectrogram( 110 const std::vector<float>& input, 111 std::vector<std::vector<std::complex<float>>>*); 112 extern template bool Spectrogram::ComputeComplexSpectrogram( 113 const std::vector<double>& input, 114 std::vector<std::vector<std::complex<float>>>*); 115 extern template bool Spectrogram::ComputeComplexSpectrogram( 116 const std::vector<float>& input, 117 std::vector<std::vector<std::complex<double>>>*); 118 extern template bool Spectrogram::ComputeComplexSpectrogram( 119 const std::vector<double>& input, 120 std::vector<std::vector<std::complex<double>>>*); 121 122 extern template bool Spectrogram::ComputeSquaredMagnitudeSpectrogram( 123 const std::vector<float>& input, std::vector<std::vector<float>>*); 124 extern template bool Spectrogram::ComputeSquaredMagnitudeSpectrogram( 125 const std::vector<double>& input, std::vector<std::vector<float>>*); 126 extern template bool Spectrogram::ComputeSquaredMagnitudeSpectrogram( 127 const std::vector<float>& input, std::vector<std::vector<double>>*); 128 extern template bool Spectrogram::ComputeSquaredMagnitudeSpectrogram( 129 const std::vector<double>& input, std::vector<std::vector<double>>*); 130 131 } // namespace internal 132 } // namespace tflite 133 134 #endif // TENSORFLOW_LITE_KERNELS_INTERNAL_SPECTROGRAM_H_ 135