1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/micro/examples/micro_speech/feature_provider.h"
17
18 #include "tensorflow/lite/micro/examples/micro_speech/audio_provider.h"
19 #include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_features_generator.h"
20 #include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h"
21
FeatureProvider(int feature_size,int8_t * feature_data)22 FeatureProvider::FeatureProvider(int feature_size, int8_t* feature_data)
23 : feature_size_(feature_size),
24 feature_data_(feature_data),
25 is_first_run_(true) {
26 // Initialize the feature data to default values.
27 for (int n = 0; n < feature_size_; ++n) {
28 feature_data_[n] = 0;
29 }
30 }
31
~FeatureProvider()32 FeatureProvider::~FeatureProvider() {}
33
PopulateFeatureData(tflite::ErrorReporter * error_reporter,int32_t last_time_in_ms,int32_t time_in_ms,int * how_many_new_slices)34 TfLiteStatus FeatureProvider::PopulateFeatureData(
35 tflite::ErrorReporter* error_reporter, int32_t last_time_in_ms,
36 int32_t time_in_ms, int* how_many_new_slices) {
37 if (feature_size_ != kFeatureElementCount) {
38 TF_LITE_REPORT_ERROR(error_reporter,
39 "Requested feature_data_ size %d doesn't match %d",
40 feature_size_, kFeatureElementCount);
41 return kTfLiteError;
42 }
43
44 // Quantize the time into steps as long as each window stride, so we can
45 // figure out which audio data we need to fetch.
46 const int last_step = (last_time_in_ms / kFeatureSliceStrideMs);
47 const int current_step = (time_in_ms / kFeatureSliceStrideMs);
48
49 int slices_needed = current_step - last_step;
50 // If this is the first call, make sure we don't use any cached information.
51 if (is_first_run_) {
52 TfLiteStatus init_status = InitializeMicroFeatures(error_reporter);
53 if (init_status != kTfLiteOk) {
54 return init_status;
55 }
56 is_first_run_ = false;
57 slices_needed = kFeatureSliceCount;
58 }
59 if (slices_needed > kFeatureSliceCount) {
60 slices_needed = kFeatureSliceCount;
61 }
62 *how_many_new_slices = slices_needed;
63
64 const int slices_to_keep = kFeatureSliceCount - slices_needed;
65 const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
66 // If we can avoid recalculating some slices, just move the existing data
67 // up in the spectrogram, to perform something like this:
68 // last time = 80ms current time = 120ms
69 // +-----------+ +-----------+
70 // | data@20ms | --> | data@60ms |
71 // +-----------+ -- +-----------+
72 // | data@40ms | -- --> | data@80ms |
73 // +-----------+ -- -- +-----------+
74 // | data@60ms | -- -- | <empty> |
75 // +-----------+ -- +-----------+
76 // | data@80ms | -- | <empty> |
77 // +-----------+ +-----------+
78 if (slices_to_keep > 0) {
79 for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
80 int8_t* dest_slice_data =
81 feature_data_ + (dest_slice * kFeatureSliceSize);
82 const int src_slice = dest_slice + slices_to_drop;
83 const int8_t* src_slice_data =
84 feature_data_ + (src_slice * kFeatureSliceSize);
85 for (int i = 0; i < kFeatureSliceSize; ++i) {
86 dest_slice_data[i] = src_slice_data[i];
87 }
88 }
89 }
90 // Any slices that need to be filled in with feature data have their
91 // appropriate audio data pulled, and features calculated for that slice.
92 if (slices_needed > 0) {
93 for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
94 ++new_slice) {
95 const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
96 const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
97 int16_t* audio_samples = nullptr;
98 int audio_samples_size = 0;
99 // TODO(petewarden): Fix bug that leads to non-zero slice_start_ms
100 GetAudioSamples(error_reporter, (slice_start_ms > 0 ? slice_start_ms : 0),
101 kFeatureSliceDurationMs, &audio_samples_size,
102 &audio_samples);
103 if (audio_samples_size < kMaxAudioSampleSize) {
104 TF_LITE_REPORT_ERROR(error_reporter,
105 "Audio data size %d too small, want %d",
106 audio_samples_size, kMaxAudioSampleSize);
107 return kTfLiteError;
108 }
109 int8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
110 size_t num_samples_read;
111 TfLiteStatus generate_status = GenerateMicroFeatures(
112 error_reporter, audio_samples, audio_samples_size, kFeatureSliceSize,
113 new_slice_data, &num_samples_read);
114 if (generate_status != kTfLiteOk) {
115 return generate_status;
116 }
117 }
118 }
119 return kTfLiteOk;
120 }
121