cast/standalone_sender/streaming_opus_encoder.cc

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "cast/standalone_sender/streaming_opus_encoder.h"

#include <opus/opus.h>

#include <algorithm>
#include <chrono>

#include "util/chrono_helpers.h"

namespace openscreen {
namespace cast {

using openscreen::operator<<;  // To pretty-print chrono values.

namespace {

// The bitrate at which virtually all stereo audio can be encoded and decoded
// without human-perceivable artifacts. Source:
// https://wiki.hydrogenaud.io/index.php?title=Opus#Bitrate_performance
constexpr opus_int32 kTransparentBitrate = 160000;

// The maximum number of Cast audio frames the encoder may fall behind by before
// skipping-ahead the RTP timestamps to compensate.
constexpr int kMaxCastFramesBeforeSkip = 3;

}  // namespace

StreamingOpusEncoder::StreamingOpusEncoder(int num_channels,
                                           int cast_frames_per_second,
                                           Sender* sender)
    : num_channels_(num_channels),
      sender_(sender),
      samples_per_cast_frame_(sample_rate() / cast_frames_per_second),
      approximate_cast_frame_duration_(Clock::to_duration(seconds(1)) /
                                       cast_frames_per_second),
      encoder_storage_(new uint8_t[opus_encoder_get_size(num_channels_)]),
      input_(new float[num_channels_ * samples_per_cast_frame_]),
      output_(new uint8_t[kOpusMaxPayloadSize]) {
  OSP_CHECK_GT(cast_frames_per_second, 0);
  OSP_DCHECK(sender_);
  OSP_CHECK_GT(samples_per_cast_frame_, 0);
  OSP_CHECK_EQ(sample_rate() % cast_frames_per_second, 0);
  OSP_CHECK(approximate_cast_frame_duration_ > Clock::duration::zero());

  frame_.dependency = EncodedFrame::KEY_FRAME;

  const auto init_result = opus_encoder_init(
      encoder(), sample_rate(), num_channels_, OPUS_APPLICATION_AUDIO);
  OSP_CHECK_EQ(init_result, OPUS_OK);

  UseStandardQuality();
}

StreamingOpusEncoder::~StreamingOpusEncoder() = default;

int StreamingOpusEncoder::GetBitrate() const {
  opus_int32 bitrate;
  const auto ctl_result =
      opus_encoder_ctl(encoder(), OPUS_GET_BITRATE(&bitrate));
  OSP_CHECK_EQ(ctl_result, OPUS_OK);
  return bitrate;
}

void StreamingOpusEncoder::UseStandardQuality() {
  const auto ctl_result =
      opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(OPUS_AUTO));
  OSP_CHECK_EQ(ctl_result, OPUS_OK);
  UpdateCodecDelay();
}

void StreamingOpusEncoder::UseHighQuality() {
  // kTransparentBitrate assumes stereo audio. Scale it by the actual number of
  // channels.
  const opus_int32 bitrate = kTransparentBitrate * num_channels_ / 2;
  const auto ctl_result =
      opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(bitrate));
  OSP_CHECK_EQ(ctl_result, OPUS_OK);
  UpdateCodecDelay();
}

void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples,
                                         int num_samples,
                                         Clock::time_point reference_time) {
  OSP_DCHECK(interleaved_samples);
  OSP_DCHECK_GT(num_samples, 0);

  ResolveTimestampsAndMaybeSkip(reference_time);

  while (num_samples > 0) {
    const int samples_copied =
        FillInputBuffer(interleaved_samples, num_samples);
    num_samples -= samples_copied;
    interleaved_samples += num_channels_ * samples_copied;

    if (num_samples_queued_ < samples_per_cast_frame_) {
      return;  // Not enough yet for a full Cast audio frame.
    }

    const opus_int32 packet_size_or_error =
        opus_encode_float(encoder(), input_.get(), num_samples_queued_,
                          output_.get(), kOpusMaxPayloadSize);
    num_samples_queued_ = 0;
    if (packet_size_or_error < 0) {
      OSP_LOG_FATAL << "AUDIO[" << sender_->ssrc()
                    << "] Error code from opus_encode_float(): "
                    << packet_size_or_error;
      return;
    }

    frame_.frame_id = sender_->GetNextFrameId();
    frame_.referenced_frame_id = frame_.frame_id;
    // Note: It's possible for Opus to encode a zero byte packet. Send a Cast
    // audio frame anyway, to represent the passage of silence and to send other
    // stream metadata.
    frame_.data = absl::Span<uint8_t>(output_.get(), packet_size_or_error);
    last_sent_frame_reference_time_ = frame_.reference_time;
    switch (sender_->EnqueueFrame(frame_)) {
      case Sender::OK:
        break;
      case Sender::PAYLOAD_TOO_LARGE:
        OSP_NOTREACHED();  // The Opus packet cannot possibly be too large.
        break;
      case Sender::REACHED_ID_SPAN_LIMIT:
        OSP_LOG_WARN << "AUDIO[" << sender_->ssrc()
                     << "] Dropping: FrameId span limit reached.";
        break;
      case Sender::MAX_DURATION_IN_FLIGHT:
        OSP_LOG_INFO << "AUDIO[" << sender_->ssrc()
                     << "] Dropping: In-flight duration would be too high.";
        break;
    }

    frame_.rtp_timestamp += RtpTimeDelta::FromTicks(samples_per_cast_frame_);
    frame_.reference_time += approximate_cast_frame_duration_;
  }
}

void StreamingOpusEncoder::UpdateCodecDelay() {
  opus_int32 lookahead = 0;
  const auto ctl_result =
      opus_encoder_ctl(encoder(), OPUS_GET_LOOKAHEAD(&lookahead));
  OSP_CHECK_EQ(ctl_result, OPUS_OK);
  codec_delay_ = RtpTimeDelta::FromTicks(lookahead).ToDuration<Clock::duration>(
      sample_rate());
}

void StreamingOpusEncoder::ResolveTimestampsAndMaybeSkip(
    Clock::time_point reference_time) {
  // Back-track the reference time to account for the audio delay introduced by
  // the codec.
  reference_time -= codec_delay_;

  // Special case: Nothing special for the first frame's timestamps.
  if (start_time_ == Clock::time_point::min()) {
    frame_.rtp_timestamp = RtpTimeTicks();
    frame_.reference_time = start_time_ = reference_time;
    last_sent_frame_reference_time_ =
        reference_time - approximate_cast_frame_duration_;
    return;
  }

  const RtpTimeTicks current_position =
      frame_.rtp_timestamp + RtpTimeDelta::FromTicks(num_samples_queued_);
  const RtpTimeTicks reference_position = RtpTimeTicks::FromTimeSinceOrigin(
      reference_time - start_time_, sample_rate());
  const RtpTimeDelta rtp_advancement = reference_position - current_position;
  const RtpTimeDelta skip_threshold =
      RtpTimeDelta::FromTicks(samples_per_cast_frame_) *
      kMaxCastFramesBeforeSkip;
  if (rtp_advancement > skip_threshold) {
    OSP_LOG_WARN << "Detected audio gap "
                 << rtp_advancement.ToDuration<microseconds>(sample_rate())
                 << ", skipping ahead...";
    num_samples_queued_ = 0;
    frame_.rtp_timestamp = reference_position;
  }

  // Further back-track the reference time to account for the already-queued
  // samples.
  reference_time -= RtpTimeDelta::FromTicks(num_samples_queued_)
                        .ToDuration<Clock::duration>(sample_rate());

  // Frame reference times must be monotonically increasing. A little noise in
  // the negative direction is okay to cap-off. Log a warning if there's a
  // bigger problem (at the source).
  const Clock::time_point lower_bound =
      last_sent_frame_reference_time_ +
      RtpTimeDelta::FromTicks(1).ToDuration<Clock::duration>(sample_rate());
  if (reference_time < lower_bound) {
    const Clock::duration backwards_amount =
        last_sent_frame_reference_time_ - reference_time;
    OSP_LOG_IF(WARN, backwards_amount >= approximate_cast_frame_duration_)
        << "Reference time went *backwards* too much (" << backwards_amount
        << " in wrong direction). A/V sync may suffer at the Receiver!";
    reference_time = lower_bound;
  }

  frame_.reference_time = reference_time;
}

int StreamingOpusEncoder::FillInputBuffer(const float* interleaved_samples,
                                          int num_samples) {
  const int samples_needed = samples_per_cast_frame_ - num_samples_queued_;
  const int samples_to_copy = std::min(num_samples, samples_needed);
  std::copy(interleaved_samples,
            interleaved_samples + num_channels_ * samples_to_copy,
            input_.get() + num_channels_ * num_samples_queued_);
  num_samples_queued_ += samples_to_copy;
  return samples_to_copy;
}

// static
constexpr int StreamingOpusEncoder::kDefaultCastAudioFramesPerSecond;
// static
constexpr int StreamingOpusEncoder::kOpusMaxPayloadSize;

}  // namespace cast
}  // namespace openscreen