1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef SDK_ANDROID_SRC_JNI_AUDIO_DEVICE_OPENSLES_PLAYER_H_ 12 #define SDK_ANDROID_SRC_JNI_AUDIO_DEVICE_OPENSLES_PLAYER_H_ 13 14 #include <SLES/OpenSLES.h> 15 #include <SLES/OpenSLES_Android.h> 16 #include <SLES/OpenSLES_AndroidConfiguration.h> 17 18 #include <memory> 19 #include "absl/types/optional.h" 20 #include "api/scoped_refptr.h" 21 #include "modules/audio_device/audio_device_buffer.h" 22 #include "modules/audio_device/fine_audio_buffer.h" 23 #include "modules/audio_device/include/audio_device_defines.h" 24 #include "rtc_base/thread_checker.h" 25 #include "sdk/android/src/jni/audio_device/audio_common.h" 26 #include "sdk/android/src/jni/audio_device/audio_device_module.h" 27 #include "sdk/android/src/jni/audio_device/opensles_common.h" 28 29 namespace webrtc { 30 31 class FineAudioBuffer; 32 33 namespace jni { 34 35 // Implements 16-bit mono PCM audio output support for Android using the 36 // C based OpenSL ES API. No calls from C/C++ to Java using JNI is done. 37 // 38 // An instance can be created on any thread, but must then be used on one and 39 // the same thread. All public methods must also be called on the same thread. A 40 // thread checker will RTC_DCHECK if any method is called on an invalid thread. 41 // Decoded audio buffers are requested on a dedicated internal thread managed by 42 // the OpenSL ES layer. 43 // 44 // The existing design forces the user to call InitPlayout() after Stoplayout() 45 // to be able to call StartPlayout() again. This is inline with how the Java- 46 // based implementation works. 47 // 48 // OpenSL ES is a native C API which have no Dalvik-related overhead such as 49 // garbage collection pauses and it supports reduced audio output latency. 50 // If the device doesn't claim this feature but supports API level 9 (Android 51 // platform version 2.3) or later, then we can still use the OpenSL ES APIs but 52 // the output latency may be higher. 53 class OpenSLESPlayer : public AudioOutput { 54 public: 55 // Beginning with API level 17 (Android 4.2), a buffer count of 2 or more is 56 // required for lower latency. Beginning with API level 18 (Android 4.3), a 57 // buffer count of 1 is sufficient for lower latency. In addition, the buffer 58 // size and sample rate must be compatible with the device's native output 59 // configuration provided via the audio manager at construction. 60 // TODO(henrika): perhaps set this value dynamically based on OS version. 61 static const int kNumOfOpenSLESBuffers = 2; 62 63 OpenSLESPlayer(const AudioParameters& audio_parameters, 64 rtc::scoped_refptr<OpenSLEngineManager> engine_manager); 65 ~OpenSLESPlayer() override; 66 67 int Init() override; 68 int Terminate() override; 69 70 int InitPlayout() override; 71 bool PlayoutIsInitialized() const override; 72 73 int StartPlayout() override; 74 int StopPlayout() override; 75 bool Playing() const override; 76 77 bool SpeakerVolumeIsAvailable() override; 78 int SetSpeakerVolume(uint32_t volume) override; 79 absl::optional<uint32_t> SpeakerVolume() const override; 80 absl::optional<uint32_t> MaxSpeakerVolume() const override; 81 absl::optional<uint32_t> MinSpeakerVolume() const override; 82 83 void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; 84 GetPlayoutUnderrunCount()85 int GetPlayoutUnderrunCount() override { return -1; } 86 87 private: 88 // These callback methods are called when data is required for playout. 89 // They are both called from an internal "OpenSL ES thread" which is not 90 // attached to the Dalvik VM. 91 static void SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf caller, 92 void* context); 93 void FillBufferQueue(); 94 // Reads audio data in PCM format using the AudioDeviceBuffer. 95 // Can be called both on the main thread (during Start()) and from the 96 // internal audio thread while output streaming is active. 97 // If the |silence| flag is set, the audio is filled with zeros instead of 98 // asking the WebRTC layer for real audio data. This procedure is also known 99 // as audio priming. 100 void EnqueuePlayoutData(bool silence); 101 102 // Allocate memory for audio buffers which will be used to render audio 103 // via the SLAndroidSimpleBufferQueueItf interface. 104 void AllocateDataBuffers(); 105 106 // Obtaines the SL Engine Interface from the existing global Engine object. 107 // The interface exposes creation methods of all the OpenSL ES object types. 108 // This method defines the |engine_| member variable. 109 bool ObtainEngineInterface(); 110 111 // Creates/destroys the output mix object. 112 bool CreateMix(); 113 void DestroyMix(); 114 115 // Creates/destroys the audio player and the simple-buffer object. 116 // Also creates the volume object. 117 bool CreateAudioPlayer(); 118 void DestroyAudioPlayer(); 119 120 SLuint32 GetPlayState() const; 121 122 // Ensures that methods are called from the same thread as this object is 123 // created on. 124 rtc::ThreadChecker thread_checker_; 125 126 // Stores thread ID in first call to SimpleBufferQueueCallback() from internal 127 // non-application thread which is not attached to the Dalvik JVM. 128 // Detached during construction of this object. 129 rtc::ThreadChecker thread_checker_opensles_; 130 131 const AudioParameters audio_parameters_; 132 133 // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the 134 // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create(). 135 AudioDeviceBuffer* audio_device_buffer_; 136 137 bool initialized_; 138 bool playing_; 139 140 // PCM-type format definition. 141 // TODO(henrika): add support for SLAndroidDataFormat_PCM_EX (android-21) if 142 // 32-bit float representation is needed. 143 SLDataFormat_PCM pcm_format_; 144 145 // Queue of audio buffers to be used by the player object for rendering 146 // audio. 147 std::unique_ptr<SLint16[]> audio_buffers_[kNumOfOpenSLESBuffers]; 148 149 // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data 150 // in chunks of 10ms. It then allows for this data to be pulled in 151 // a finer or coarser granularity. I.e. interacting with this class instead 152 // of directly with the AudioDeviceBuffer one can ask for any number of 153 // audio data samples. 154 // Example: native buffer size can be 192 audio frames at 48kHz sample rate. 155 // WebRTC will provide 480 audio frames per 10ms but OpenSL ES asks for 192 156 // in each callback (one every 4th ms). This class can then ask for 192 and 157 // the FineAudioBuffer will ask WebRTC for new data approximately only every 158 // second callback and also cache non-utilized audio. 159 std::unique_ptr<FineAudioBuffer> fine_audio_buffer_; 160 161 // Keeps track of active audio buffer 'n' in the audio_buffers_[n] queue. 162 // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... 163 int buffer_index_; 164 165 const rtc::scoped_refptr<OpenSLEngineManager> engine_manager_; 166 // This interface exposes creation methods for all the OpenSL ES object types. 167 // It is the OpenSL ES API entry point. 168 SLEngineItf engine_; 169 170 // Output mix object to be used by the player object. 171 ScopedSLObjectItf output_mix_; 172 173 // The audio player media object plays out audio to the speakers. It also 174 // supports volume control. 175 ScopedSLObjectItf player_object_; 176 177 // This interface is supported on the audio player and it controls the state 178 // of the audio player. 179 SLPlayItf player_; 180 181 // The Android Simple Buffer Queue interface is supported on the audio player 182 // and it provides methods to send audio data from the source to the audio 183 // player for rendering. 184 SLAndroidSimpleBufferQueueItf simple_buffer_queue_; 185 186 // This interface exposes controls for manipulating the object’s audio volume 187 // properties. This interface is supported on the Audio Player object. 188 SLVolumeItf volume_; 189 190 // Last time the OpenSL ES layer asked for audio data to play out. 191 uint32_t last_play_time_; 192 }; 193 194 } // namespace jni 195 196 } // namespace webrtc 197 198 #endif // SDK_ANDROID_SRC_JNI_AUDIO_DEVICE_OPENSLES_PLAYER_H_ 199