1 /*
2 * Copyright (c) 2023, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 3-Clause Clear License
5 * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
6 * License was not distributed with this source code in the LICENSE file, you
7 * can obtain it at www.aomedia.org/license/software-license/bsd-3-c-c. If the
8 * Alliance for Open Media Patent License 1.0 was not distributed with this
9 * source code in the PATENTS file, you can obtain it at
10 * www.aomedia.org/license/patent.
11 */
12 #include "iamf/cli/codec/opus_encoder.h"
13
14 #include <cstddef>
15 #include <cstdint>
16 #include <memory>
17 #include <utility>
18 #include <vector>
19
20 #include "absl/functional/any_invocable.h"
21 #include "absl/log/log.h"
22 #include "absl/status/status.h"
23 #include "absl/strings/str_cat.h"
24 #include "absl/synchronization/mutex.h"
25 #include "absl/types/span.h"
26 #include "iamf/cli/audio_frame_with_data.h"
27 #include "iamf/cli/codec/opus_utils.h"
28 #include "iamf/cli/proto/codec_config.pb.h"
29 #include "iamf/common/utils/macros.h"
30 #include "iamf/common/utils/numeric_utils.h"
31 #include "iamf/common/utils/sample_processing_utils.h"
32 #include "iamf/common/utils/validation_utils.h"
33 #include "iamf/obu/decoder_config/opus_decoder_config.h"
34 #include "include/opus.h"
35 #include "include/opus_defines.h"
36 #include "include/opus_types.h"
37
38 namespace iamf_tools {
39
40 namespace {
41
42 // Performs validation for values that this implementation assumes are
43 // restricted because they are restricted in IAMF v1.1.0.
ValidateDecoderConfig(const OpusDecoderConfig & opus_decoder_config)44 absl::Status ValidateDecoderConfig(
45 const OpusDecoderConfig& opus_decoder_config) {
46 // Validate the input. Reject values that would need to be added to this
47 // function if they were ever supported.
48 if (opus_decoder_config.output_gain_ != 0 ||
49 opus_decoder_config.mapping_family_ != 0) {
50 auto error_message = absl::StrCat(
51 "IAMF v1.1.0 expects output_gain: ", opus_decoder_config.output_gain_,
52 " and mapping_family: ", opus_decoder_config.mapping_family_,
53 " to be 0.");
54 return absl::InvalidArgumentError(error_message);
55 }
56
57 return absl::OkStatus();
58 }
59
60 // `opus_encode_float` recommends the input is normalized to the range [-1, 1].
61 const absl::AnyInvocable<absl::Status(int32_t, float&) const>
__anon0c89415d0202(int32_t input, float& output) 62 kInt32ToNormalizedFloat = [](int32_t input, float& output) {
63 output = Int32ToNormalizedFloatingPoint<float>(input);
64 return absl::OkStatus();
65 };
66
EncodeFloat(const std::vector<std::vector<int32_t>> & samples,int num_samples_per_channel,::OpusEncoder * encoder,std::vector<uint8_t> & audio_frame)67 absl::StatusOr<int> EncodeFloat(
68 const std::vector<std::vector<int32_t>>& samples,
69 int num_samples_per_channel, ::OpusEncoder* encoder,
70 std::vector<uint8_t>& audio_frame) {
71 std::vector<float> encoder_input_pcm;
72 RETURN_IF_NOT_OK(ConvertTimeChannelToInterleaved(absl::MakeConstSpan(samples),
73 kInt32ToNormalizedFloat,
74 encoder_input_pcm));
75
76 // TODO(b/311655037): Test that samples are passed to `opus_encode_float` in
77 // the correct order. Maybe also check they are in the
78 // correct [-1, +1] range. This may requiring mocking a
79 // simple version of `opus_encode_float`.
80 return opus_encode_float(encoder, encoder_input_pcm.data(),
81 num_samples_per_channel, audio_frame.data(),
82 static_cast<opus_int32>(audio_frame.size()));
83 }
84
EncodeInt16(const std::vector<std::vector<int32_t>> & samples,int num_samples_per_channel,int num_channels,::OpusEncoder * encoder,std::vector<uint8_t> & audio_frame)85 absl::StatusOr<int> EncodeInt16(
86 const std::vector<std::vector<int32_t>>& samples,
87 int num_samples_per_channel, int num_channels, ::OpusEncoder* encoder,
88 std::vector<uint8_t>& audio_frame) {
89 // `libopus` requires the native system endianness as input.
90 const bool big_endian = IsNativeBigEndian();
91 // Convert input to the array that will be passed to `opus_encode`.
92 std::vector<opus_int16> encoder_input_pcm(
93 num_samples_per_channel * num_channels, 0);
94 size_t write_position = 0;
95 for (int t = 0; t < samples.size(); t++) {
96 for (int c = 0; c < samples[0].size(); ++c) {
97 // Convert all frames to 16-bit samples for input to Opus.
98 // Write the 16-bit samples directly into the pcm vector.
99 RETURN_IF_NOT_OK(
100 WritePcmSample(static_cast<uint32_t>(samples[t][c]), 16, big_endian,
101 reinterpret_cast<uint8_t*>(encoder_input_pcm.data()),
102 write_position));
103 }
104 }
105
106 return opus_encode(encoder, encoder_input_pcm.data(), num_samples_per_channel,
107 audio_frame.data(),
108 static_cast<opus_int32>(audio_frame.size()));
109 }
110
111 } // namespace
112
SetNumberOfSamplesToDelayAtStart(bool validate_codec_delay)113 absl::Status OpusEncoder::SetNumberOfSamplesToDelayAtStart(
114 bool validate_codec_delay) {
115 opus_int32 lookahead;
116 opus_encoder_ctl(encoder_, OPUS_GET_LOOKAHEAD(&lookahead));
117 LOG_FIRST_N(INFO, 1) << "Opus lookahead=" << lookahead;
118 // Opus calls the number of samples that should be trimmed/pre-skipped
119 // `lookahead`.
120 required_samples_to_delay_at_start_ = static_cast<uint32_t>(lookahead);
121 if (validate_codec_delay) {
122 MAYBE_RETURN_IF_NOT_OK(
123 ValidateEqual(static_cast<uint32_t>(decoder_config_.pre_skip_),
124 required_samples_to_delay_at_start_, "Opus `pre_skip`"));
125 }
126
127 return absl::OkStatus();
128 }
129
InitializeEncoder()130 absl::Status OpusEncoder::InitializeEncoder() {
131 MAYBE_RETURN_IF_NOT_OK(ValidateDecoderConfig(decoder_config_));
132
133 int application;
134 switch (encoder_metadata_.application()) {
135 using enum iamf_tools_cli_proto::OpusApplicationFlag;
136 case APPLICATION_VOIP:
137 application = OPUS_APPLICATION_VOIP;
138 break;
139 case APPLICATION_AUDIO:
140 application = OPUS_APPLICATION_AUDIO;
141 break;
142 case APPLICATION_RESTRICTED_LOWDELAY:
143 application = OPUS_APPLICATION_RESTRICTED_LOWDELAY;
144 break;
145 default:
146 return absl::InvalidArgumentError(absl::StrCat(
147 "Unrecognized application.", encoder_metadata_.application()));
148 }
149
150 int opus_error_code;
151 encoder_ = opus_encoder_create(input_sample_rate_, num_channels_, application,
152 &opus_error_code);
153 RETURN_IF_NOT_OK(OpusErrorCodeToAbslStatus(
154 opus_error_code, "Failed to initialize Opus encoder."));
155
156 // `OPUS_SET_BITRATE` treats this as the bit-rate for the entire substream.
157 // Configure `libopus` so coupled substreams and mono substreams have equally
158 // effective bit-rate per channel.
159 float opus_rate;
160 if (encoder_metadata_.substream_id_to_bitrate_override().contains(
161 substream_id_)) {
162 opus_rate =
163 encoder_metadata_.substream_id_to_bitrate_override().at(substream_id_);
164 } else if (num_channels_ > 1) {
165 opus_rate = encoder_metadata_.target_bitrate_per_channel() * num_channels_ *
166 encoder_metadata_.coupling_rate_adjustment();
167 } else {
168 opus_rate = encoder_metadata_.target_bitrate_per_channel();
169 }
170 opus_encoder_ctl(encoder_,
171 OPUS_SET_BITRATE(static_cast<opus_int32>(opus_rate + 0.5f)));
172
173 return absl::OkStatus();
174 }
175
~OpusEncoder()176 OpusEncoder::~OpusEncoder() { opus_encoder_destroy(encoder_); }
177
EncodeAudioFrame(int,const std::vector<std::vector<int32_t>> & samples,std::unique_ptr<AudioFrameWithData> partial_audio_frame_with_data)178 absl::Status OpusEncoder::EncodeAudioFrame(
179 int /*input_bit_depth*/, const std::vector<std::vector<int32_t>>& samples,
180 std::unique_ptr<AudioFrameWithData> partial_audio_frame_with_data) {
181 RETURN_IF_NOT_OK(ValidateNotFinalized());
182 RETURN_IF_NOT_OK(ValidateInputSamples(samples));
183 const int num_samples_per_channel = static_cast<int>(num_samples_per_frame_);
184
185 // Opus output could take up to 4 bytes per sample. Reserve an output vector
186 // of the maximum possible size.
187 auto& audio_frame = partial_audio_frame_with_data->obu.audio_frame_;
188 audio_frame.resize(num_samples_per_channel * num_channels_ * 4, 0);
189
190 const auto encoded_length_bytes =
191 encoder_metadata_.use_float_api()
192 ? EncodeFloat(samples, num_samples_per_channel, encoder_, audio_frame)
193 : EncodeInt16(samples, num_samples_per_channel, num_channels_,
194 encoder_, audio_frame);
195
196 if (!encoded_length_bytes.ok()) {
197 return encoded_length_bytes.status();
198 }
199
200 if (*encoded_length_bytes < 0) {
201 // When `encoded_length_bytes` is negative, it is a non-OK Opus error code.
202 return OpusErrorCodeToAbslStatus(*encoded_length_bytes,
203 "Failed to encode samples.");
204 }
205
206 // Shrink output vector to actual size.
207 audio_frame.resize(*encoded_length_bytes);
208
209 absl::MutexLock lock(&mutex_);
210 finalized_audio_frames_.emplace_back(
211 std::move(*partial_audio_frame_with_data));
212
213 return absl::OkStatus();
214 }
215
216 } // namespace iamf_tools
217