1# Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""AudioMicrofrontend Op creates filterbanks from audio data.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21from tensorflow.lite.experimental.microfrontend.ops import gen_audio_microfrontend_op 22from tensorflow.python.framework import dtypes 23from tensorflow.python.framework import load_library 24from tensorflow.python.framework import ops 25from tensorflow.python.ops import array_ops 26from tensorflow.python.platform import resource_loader 27from tensorflow.python.util.tf_export import tf_export 28 29_audio_microfrontend_op = load_library.load_op_library( 30 resource_loader.get_path_to_datafile("_audio_microfrontend_op.so")) 31 32 33@tf_export("lite.experimental.microfrontend.python.ops.audio_microfrontend") 34def audio_microfrontend(audio, 35 sample_rate=16000, 36 window_size=25, 37 window_step=10, 38 num_channels=32, 39 upper_band_limit=7500.0, 40 lower_band_limit=125.0, 41 smoothing_bits=10, 42 even_smoothing=0.025, 43 odd_smoothing=0.06, 44 min_signal_remaining=0.05, 45 enable_pcan=True, 46 pcan_strength=0.95, 47 pcan_offset=80.0, 48 gain_bits=21, 49 enable_log=True, 50 scale_shift=6, 51 left_context=0, 52 right_context=0, 53 frame_stride=1, 54 zero_padding=False, 55 out_scale=1, 56 out_type=dtypes.uint16): 57 """Audio Microfrontend Op. 58 59 This Op converts a sequence of audio data into one or more 60 feature vectors containing filterbanks of the input. The 61 conversion process uses a lightweight library to perform: 62 63 1. A slicing window function 64 2. Short-time FFTs 65 3. Filterbank calculations 66 4. Noise reduction 67 5. PCAN Auto Gain Control 68 6. Logarithmic scaling 69 70 Args: 71 audio: 1D Tensor, int16 audio data in temporal ordering. 72 sample_rate: Integer, the sample rate of the audio in Hz. 73 window_size: Integer, length of desired time frames in ms. 74 window_step: Integer, length of step size for the next frame in ms. 75 num_channels: Integer, the number of filterbank channels to use. 76 upper_band_limit: Float, the highest frequency included in the filterbanks. 77 lower_band_limit: Float, the lowest frequency included in the filterbanks. 78 smoothing_bits: Int, scale up signal by 2^(smoothing_bits) before reduction. 79 even_smoothing: Float, smoothing coefficient for even-numbered channels. 80 odd_smoothing: Float, smoothing coefficient for odd-numbered channels. 81 min_signal_remaining: Float, fraction of signal to preserve in smoothing. 82 enable_pcan: Bool, enable PCAN auto gain control. 83 pcan_strength: Float, gain normalization exponent. 84 pcan_offset: Float, positive value added in the normalization denominator. 85 gain_bits: Int, number of fractional bits in the gain. 86 enable_log: Bool, enable logarithmic scaling of filterbanks. 87 scale_shift: Integer, scale filterbanks by 2^(scale_shift). 88 left_context: Integer, number of preceding frames to attach to each frame. 89 right_context: Integer, number of preceding frames to attach to each frame. 90 frame_stride: Integer, M frames to skip over, where output[n] = frame[n*M]. 91 zero_padding: Bool, if left/right context is out-of-bounds, attach frame of 92 zeroes. Otherwise, frame[0] or frame[size-1] will be copied. 93 out_scale: Integer, divide all filterbanks by this number. 94 out_type: DType, type of the output Tensor, defaults to UINT16. 95 96 Returns: 97 filterbanks: 2D Tensor, each row is a time frame, each column is a channel. 98 99 Raises: 100 ValueError: If the audio tensor is not explicitly a vector. 101 """ 102 audio_shape = audio.shape 103 if audio_shape.ndims is None: 104 raise ValueError("Input to `AudioMicrofrontend` should have known rank.") 105 if len(audio_shape) > 1: 106 audio = array_ops.reshape(audio, [-1]) 107 108 return gen_audio_microfrontend_op.audio_microfrontend( 109 audio, sample_rate, window_size, window_step, num_channels, 110 upper_band_limit, lower_band_limit, smoothing_bits, even_smoothing, 111 odd_smoothing, min_signal_remaining, enable_pcan, pcan_strength, 112 pcan_offset, gain_bits, enable_log, scale_shift, left_context, 113 right_context, frame_stride, zero_padding, out_scale, out_type) 114 115 116ops.NotDifferentiable("AudioMicrofrontend") 117