1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #if defined(ARDUINO) && !defined(ARDUINO_SFE_EDGE)
17 #define ARDUINO_EXCLUDE_CODE
18 #endif // defined(ARDUINO) && !defined(ARDUINO_SFE_EDGE)
19
20 #ifndef ARDUINO_EXCLUDE_CODE
21
22 #include "tensorflow/lite/micro/examples/micro_speech/audio_provider.h"
23
24 #include <limits>
25
26 // These are headers from Ambiq's Apollo3 SDK.
27 #include "am_bsp.h" // NOLINT
28 #include "am_mcu_apollo.h" // NOLINT
29 #include "am_util.h" // NOLINT
30 #include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h"
31
32 namespace {
33
34 // These are the raw buffers that are filled by the ADC during DMA
35 constexpr int kAdcNumSlots = 2;
36 constexpr int kAdcSamplesPerSlot = 1024;
37 constexpr int kAdcSampleBufferSize = (kAdcNumSlots * kAdcSamplesPerSlot);
38 uint32_t g_ui32ADCSampleBuffer0[kAdcSampleBufferSize];
39 uint32_t g_ui32ADCSampleBuffer1[kAdcSampleBufferSize];
40 // Controls the double buffering between the two DMA buffers.
41 int g_dma_destination_index = 0;
42 // ADC Device Handle.
43 static void* g_adc_handle;
44 // ADC DMA error flag.
45 volatile bool g_adc_dma_error;
46 // So the interrupt can use the passed-in error handler to report issues.
47 tflite::ErrorReporter* g_adc_dma_error_reporter = nullptr;
48
49 // Holds a longer history of audio samples in a ring buffer.
50 constexpr int kAudioCaptureBufferSize = 16000;
51 int16_t g_audio_capture_buffer[kAudioCaptureBufferSize] = {};
52 int g_audio_capture_buffer_start = 0;
53 int64_t g_total_samples_captured = 0;
54 int32_t g_latest_audio_timestamp = 0;
55
56 // Copy of audio samples returned to the caller.
57 int16_t g_audio_output_buffer[kMaxAudioSampleSize];
58 bool g_is_audio_initialized = false;
59
60 // Start the DMA fetch of ADC samples.
adc_start_dma(tflite::ErrorReporter * error_reporter)61 void adc_start_dma(tflite::ErrorReporter* error_reporter) {
62 am_hal_adc_dma_config_t ADCDMAConfig;
63
64 // Configure the ADC to use DMA for the sample transfer.
65 ADCDMAConfig.bDynamicPriority = true;
66 ADCDMAConfig.ePriority = AM_HAL_ADC_PRIOR_SERVICE_IMMED;
67 ADCDMAConfig.bDMAEnable = true;
68 ADCDMAConfig.ui32SampleCount = kAdcSampleBufferSize;
69 if (g_dma_destination_index == 0) {
70 ADCDMAConfig.ui32TargetAddress = (uint32_t)g_ui32ADCSampleBuffer0;
71 } else {
72 ADCDMAConfig.ui32TargetAddress = (uint32_t)g_ui32ADCSampleBuffer1;
73 }
74 if (AM_HAL_STATUS_SUCCESS !=
75 am_hal_adc_configure_dma(g_adc_handle, &ADCDMAConfig)) {
76 TF_LITE_REPORT_ERROR(error_reporter, "Error - configuring ADC DMA failed.");
77 }
78
79 // Reset the ADC DMA flags.
80 g_adc_dma_error = false;
81 g_adc_dma_error_reporter = error_reporter;
82 }
83
84 // Configure the ADC.
adc_config0(tflite::ErrorReporter * error_reporter)85 void adc_config0(tflite::ErrorReporter* error_reporter) {
86 am_hal_adc_config_t ADCConfig;
87 am_hal_adc_slot_config_t ADCSlotConfig;
88
89 // Initialize the ADC and get the handle.
90 if (AM_HAL_STATUS_SUCCESS != am_hal_adc_initialize(0, &g_adc_handle)) {
91 TF_LITE_REPORT_ERROR(error_reporter,
92 "Error - reservation of the ADC0 instance failed.");
93 }
94
95 // Power on the ADC.
96 if (AM_HAL_STATUS_SUCCESS !=
97 am_hal_adc_power_control(g_adc_handle, AM_HAL_SYSCTRL_WAKE, false)) {
98 TF_LITE_REPORT_ERROR(error_reporter, "Error - ADC0 power on failed.");
99 }
100
101 // Set up the ADC configuration parameters. These settings are reasonable
102 // for accurate measurements at a low sample rate.
103 ADCConfig.eClock = AM_HAL_ADC_CLKSEL_HFRC_DIV2;
104 ADCConfig.ePolarity = AM_HAL_ADC_TRIGPOL_RISING;
105 ADCConfig.eTrigger = AM_HAL_ADC_TRIGSEL_SOFTWARE;
106 ADCConfig.eReference =
107 AM_HAL_ADC_REFSEL_INT_2P0; // AM_HAL_ADC_REFSEL_INT_1P5;
108 ADCConfig.eClockMode = AM_HAL_ADC_CLKMODE_LOW_LATENCY;
109 ADCConfig.ePowerMode = AM_HAL_ADC_LPMODE0;
110 ADCConfig.eRepeat = AM_HAL_ADC_REPEATING_SCAN;
111 if (AM_HAL_STATUS_SUCCESS != am_hal_adc_configure(g_adc_handle, &ADCConfig)) {
112 TF_LITE_REPORT_ERROR(error_reporter, "Error - configuring ADC0 failed.");
113 }
114
115 // Set up an ADC slot (2)
116 ADCSlotConfig.eMeasToAvg = AM_HAL_ADC_SLOT_AVG_1;
117 ADCSlotConfig.ePrecisionMode = AM_HAL_ADC_SLOT_14BIT;
118 ADCSlotConfig.eChannel = AM_HAL_ADC_SLOT_CHSEL_SE2;
119 ADCSlotConfig.bWindowCompare = false;
120 ADCSlotConfig.bEnabled = true;
121 if (AM_HAL_STATUS_SUCCESS !=
122 am_hal_adc_configure_slot(g_adc_handle, 2, &ADCSlotConfig)) {
123 TF_LITE_REPORT_ERROR(error_reporter,
124 "Error - configuring ADC Slot 2 failed.");
125 }
126
127 // Set up an ADC slot (1)
128 ADCSlotConfig.eMeasToAvg = AM_HAL_ADC_SLOT_AVG_1;
129 ADCSlotConfig.ePrecisionMode = AM_HAL_ADC_SLOT_14BIT;
130 ADCSlotConfig.eChannel = AM_HAL_ADC_SLOT_CHSEL_SE1;
131 ADCSlotConfig.bWindowCompare = false;
132 ADCSlotConfig.bEnabled = true;
133 if (AM_HAL_STATUS_SUCCESS !=
134 am_hal_adc_configure_slot(g_adc_handle, 1, &ADCSlotConfig)) {
135 TF_LITE_REPORT_ERROR(error_reporter,
136 "Error - configuring ADC Slot 1 failed.");
137 }
138
139 // Configure the ADC to use DMA for the sample transfer.
140 adc_start_dma(error_reporter);
141
142 // For this example, the samples will be coming in slowly. This means we
143 // can afford to wake up for every conversion.
144 am_hal_adc_interrupt_enable(g_adc_handle,
145 AM_HAL_ADC_INT_DERR | AM_HAL_ADC_INT_DCMP);
146
147 // Enable the ADC.
148 if (AM_HAL_STATUS_SUCCESS != am_hal_adc_enable(g_adc_handle)) {
149 TF_LITE_REPORT_ERROR(error_reporter, "Error - enabling ADC0 failed.");
150 }
151 }
152
153 // Initialize the ADC repetitive sample timer A3.
init_timerA3_for_ADC()154 void init_timerA3_for_ADC() {
155 // Start a timer to trigger the ADC periodically (1 second).
156 am_hal_ctimer_config_single(3, AM_HAL_CTIMER_TIMERA,
157 AM_HAL_CTIMER_HFRC_12MHZ |
158 AM_HAL_CTIMER_FN_REPEAT |
159 AM_HAL_CTIMER_INT_ENABLE);
160
161 am_hal_ctimer_int_enable(AM_HAL_CTIMER_INT_TIMERA3);
162
163 // 750 = 12,000,000 (clock rate) / 16,000 (desired sample rate).
164 am_hal_ctimer_period_set(3, AM_HAL_CTIMER_TIMERA, 750, 374);
165
166 // Enable the timer A3 to trigger the ADC directly
167 am_hal_ctimer_adc_trigger_enable();
168
169 // Start the timer.
170 am_hal_ctimer_start(3, AM_HAL_CTIMER_TIMERA);
171 }
172
173 // Make sure the CPU is running as fast as possible.
enable_burst_mode(tflite::ErrorReporter * error_reporter)174 void enable_burst_mode(tflite::ErrorReporter* error_reporter) {
175 am_hal_burst_avail_e eBurstModeAvailable;
176 am_hal_burst_mode_e eBurstMode;
177
178 // Check that the Burst Feature is available.
179 if (AM_HAL_STATUS_SUCCESS ==
180 am_hal_burst_mode_initialize(&eBurstModeAvailable)) {
181 if (AM_HAL_BURST_AVAIL == eBurstModeAvailable) {
182 TF_LITE_REPORT_ERROR(error_reporter, "Apollo3 Burst Mode is Available\n");
183 } else {
184 TF_LITE_REPORT_ERROR(error_reporter,
185 "Apollo3 Burst Mode is Not Available\n");
186 }
187 } else {
188 TF_LITE_REPORT_ERROR(error_reporter,
189 "Failed to Initialize for Burst Mode operation\n");
190 }
191
192 // Put the MCU into "Burst" mode.
193 if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_enable(&eBurstMode)) {
194 if (AM_HAL_BURST_MODE == eBurstMode) {
195 TF_LITE_REPORT_ERROR(error_reporter,
196 "Apollo3 operating in Burst Mode (96MHz)\n");
197 }
198 } else {
199 TF_LITE_REPORT_ERROR(error_reporter,
200 "Failed to Enable Burst Mode operation\n");
201 }
202 }
203
204 } // namespace
205
206 // Interrupt handler for the ADC.
am_adc_isr(void)207 extern "C" void am_adc_isr(void) {
208 uint32_t ui32IntMask;
209
210 // Read the interrupt status.
211 if (AM_HAL_STATUS_SUCCESS !=
212 am_hal_adc_interrupt_status(g_adc_handle, &ui32IntMask, false)) {
213 TF_LITE_REPORT_ERROR(g_adc_dma_error_reporter,
214 "Error reading ADC0 interrupt status.");
215 }
216
217 // Clear the ADC interrupt.
218 if (AM_HAL_STATUS_SUCCESS !=
219 am_hal_adc_interrupt_clear(g_adc_handle, ui32IntMask)) {
220 TF_LITE_REPORT_ERROR(g_adc_dma_error_reporter,
221 "Error clearing ADC0 interrupt status.");
222 }
223
224 // If we got a DMA complete, set the flag.
225 if (ui32IntMask & AM_HAL_ADC_INT_DCMP) {
226 uint32_t* source_buffer;
227 if (g_dma_destination_index == 0) {
228 source_buffer = g_ui32ADCSampleBuffer0;
229 g_dma_destination_index = 1;
230 } else {
231 source_buffer = g_ui32ADCSampleBuffer1;
232 g_dma_destination_index = 0;
233 }
234 adc_start_dma(g_adc_dma_error_reporter);
235
236 // For slot 1:
237 uint32_t slotCount = 0;
238 for (uint32_t indi = 0; indi < kAdcSampleBufferSize; indi++) {
239 am_hal_adc_sample_t temp;
240
241 temp.ui32Slot = AM_HAL_ADC_FIFO_SLOT(source_buffer[indi]);
242 temp.ui32Sample = AM_HAL_ADC_FIFO_SAMPLE(source_buffer[indi]);
243
244 if (temp.ui32Slot == 1) {
245 g_audio_capture_buffer[g_audio_capture_buffer_start] = temp.ui32Sample;
246 g_audio_capture_buffer_start =
247 (g_audio_capture_buffer_start + 1) % kAudioCaptureBufferSize;
248 slotCount++;
249 }
250 }
251
252 g_total_samples_captured += slotCount;
253 g_latest_audio_timestamp =
254 (g_total_samples_captured / (kAudioSampleFrequency / 1000));
255 }
256
257 // If we got a DMA error, set the flag.
258 if (ui32IntMask & AM_HAL_ADC_INT_DERR) {
259 g_adc_dma_error = true;
260 }
261 }
262
InitAudioRecording(tflite::ErrorReporter * error_reporter)263 TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
264 // Set the clock frequency.
265 if (AM_HAL_STATUS_SUCCESS !=
266 am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0)) {
267 TF_LITE_REPORT_ERROR(error_reporter,
268 "Error - configuring the system clock failed.");
269 return kTfLiteError;
270 }
271
272 // Set the default cache configuration and enable it.
273 if (AM_HAL_STATUS_SUCCESS !=
274 am_hal_cachectrl_config(&am_hal_cachectrl_defaults)) {
275 TF_LITE_REPORT_ERROR(error_reporter,
276 "Error - configuring the system cache failed.");
277 return kTfLiteError;
278 }
279 if (AM_HAL_STATUS_SUCCESS != am_hal_cachectrl_enable()) {
280 TF_LITE_REPORT_ERROR(error_reporter,
281 "Error - enabling the system cache failed.");
282 return kTfLiteError;
283 }
284
285 // Ensure the CPU is running as fast as possible.
286 enable_burst_mode(error_reporter);
287
288 // Start the CTIMER A3 for timer-based ADC measurements.
289 init_timerA3_for_ADC();
290
291 // Enable interrupts.
292 NVIC_EnableIRQ(ADC_IRQn);
293 am_hal_interrupt_master_enable();
294
295 // Edge Board Pin Definitions
296 constexpr int kSfEdgePinMic0 = 11;
297 const am_hal_gpio_pincfg_t g_sf_edge_pin_mic0 = {
298 .uFuncSel = AM_HAL_PIN_11_ADCSE2,
299 };
300 constexpr int kSfEdgePinMic1 = 29;
301 const am_hal_gpio_pincfg_t g_sf_edge_pin_mic1 = {
302 .uFuncSel = AM_HAL_PIN_29_ADCSE1,
303 };
304
305 // Set pins to act as our ADC input
306 am_hal_gpio_pinconfig(kSfEdgePinMic0, g_sf_edge_pin_mic0);
307 am_hal_gpio_pinconfig(kSfEdgePinMic1, g_sf_edge_pin_mic1);
308
309 // Configure the ADC
310 adc_config0(error_reporter);
311
312 // Trigger the ADC sampling for the first time manually.
313 if (AM_HAL_STATUS_SUCCESS != am_hal_adc_sw_trigger(g_adc_handle)) {
314 TF_LITE_REPORT_ERROR(error_reporter, "Error - triggering the ADC0 failed.");
315 return kTfLiteError;
316 }
317
318 // Enable the LED outputs.
319 am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_RED, g_AM_HAL_GPIO_OUTPUT_12);
320 am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_YELLOW, g_AM_HAL_GPIO_OUTPUT_12);
321
322 am_hal_gpio_output_set(AM_BSP_GPIO_LED_RED);
323 am_hal_gpio_output_set(AM_BSP_GPIO_LED_YELLOW);
324
325 return kTfLiteOk;
326 }
327
GetAudioSamples(tflite::ErrorReporter * error_reporter,int start_ms,int duration_ms,int * audio_samples_size,int16_t ** audio_samples)328 TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
329 int start_ms, int duration_ms,
330 int* audio_samples_size, int16_t** audio_samples) {
331 if (!g_is_audio_initialized) {
332 TfLiteStatus init_status = InitAudioRecording(error_reporter);
333 if (init_status != kTfLiteOk) {
334 return init_status;
335 }
336 g_is_audio_initialized = true;
337 }
338
339 // This is the 'zero' level of the microphone when no audio is present, and
340 // should be recalibrated if the hardware configuration ever changes. It was
341 // generated experimentally by averaging some samples captured on a board.
342 const int16_t kAdcSampleDC = 6003;
343
344 // Temporary gain emulation to deal with too-quiet audio on prototype boards.
345 const int16_t kAdcSampleGain = 10;
346
347 // This should only be called when the main thread notices that the latest
348 // audio sample data timestamp has changed, so that there's new data in the
349 // capture ring buffer. The ring buffer will eventually wrap around and
350 // overwrite the data, but the assumption is that the main thread is checking
351 // often enough and the buffer is large enough that this call will be made
352 // before that happens.
353 const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
354 const int duration_sample_count =
355 duration_ms * (kAudioSampleFrequency / 1000);
356 for (int i = 0; i < duration_sample_count; ++i) {
357 const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
358 const int32_t capture_value = g_audio_capture_buffer[capture_index];
359 int32_t output_value = capture_value - kAdcSampleDC;
360 output_value *= kAdcSampleGain;
361 if (output_value < std::numeric_limits<int16_t>::min()) {
362 output_value = std::numeric_limits<int16_t>::min();
363 }
364 if (output_value > std::numeric_limits<int16_t>::max()) {
365 output_value = std::numeric_limits<int16_t>::max();
366 }
367 g_audio_output_buffer[i] = output_value;
368 }
369
370 *audio_samples_size = kMaxAudioSampleSize;
371 *audio_samples = g_audio_output_buffer;
372 return kTfLiteOk;
373 }
374
LatestAudioTimestamp()375 int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
376
377 #endif // ARDUINO_EXCLUDE_CODE
378