1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/frontend/webrtc/lib/local_recorder.h"
18
19 #include <atomic>
20 #include <chrono>
21 #include <list>
22 #include <mutex>
23 #include <thread>
24 #include <vector>
25
26 #include <android-base/logging.h>
27 #include <api/media_stream_interface.h>
28 #include <api/rtp_parameters.h>
29 #include <api/task_queue/default_task_queue_factory.h>
30 #include <api/video/builtin_video_bitrate_allocator_factory.h>
31 #include <api/video/video_stream_encoder_create.h>
32 #include <api/video/video_stream_encoder_interface.h>
33 #include <api/video_codecs/builtin_video_encoder_factory.h>
34 #include <mkvmuxer/mkvmuxer.h>
35 #include <mkvmuxer/mkvwriter.h>
36 #include <system_wrappers/include/clock.h>
37
38 namespace cuttlefish {
39 namespace webrtc_streaming {
40
41 constexpr double kRtpTicksPerSecond = 90000.;
42 constexpr double kRtpTicksPerMs = kRtpTicksPerSecond / 1000.;
43 constexpr double kRtpTicksPerUs = kRtpTicksPerMs / 1000.;
44 constexpr double kRtpTicksPerNs = kRtpTicksPerUs / 1000.;
45
46 class LocalRecorder::Display
47 : public webrtc::EncodedImageCallback
48 , public rtc::VideoSinkInterface<webrtc::VideoFrame> {
49 public:
50 Display(LocalRecorder::Impl& impl);
51
52 void EncoderLoop();
53 void Stop();
54
55 // VideoSinkInterface
56 virtual void OnFrame(const webrtc::VideoFrame& frame) override;
57
58 // EncodedImageCallback
59 virtual webrtc::EncodedImageCallback::Result OnEncodedImage(
60 const webrtc::EncodedImage& encoded_image,
61 const webrtc::CodecSpecificInfo* codec_specific_info,
62 const webrtc::RTPFragmentationHeader* fragmentation) override;
63
64 LocalRecorder::Impl& impl_;
65 std::shared_ptr<webrtc::VideoTrackSourceInterface> source_;
66 std::unique_ptr<webrtc::VideoEncoder> video_encoder_;
67 uint64_t video_track_number_;
68
69 // TODO(schuffelen): Use a WebRTC task queue?
70 std::thread encoder_thread_;
71 std::condition_variable encoder_queue_signal_;
72 std::mutex encode_queue_mutex_;
73 std::list<webrtc::VideoFrame> encode_queue_;
74 std::atomic_bool encoder_running_ = true;
75 };
76
77 class LocalRecorder::Impl {
78 public:
79 mkvmuxer::MkvWriter file_writer_;
80 mkvmuxer::Segment segment_;
81 std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
82 std::mutex mkv_mutex_;
83 std::vector<std::unique_ptr<Display>> displays_;
84 };
85
86 /* static */
Create(const std::string & filename)87 std::unique_ptr<LocalRecorder> LocalRecorder::Create(
88 const std::string& filename) {
89 std::unique_ptr<Impl> impl(new Impl());
90
91 if (!impl->file_writer_.Open(filename.c_str())) {
92 LOG(ERROR) << "Failed to open \"" << filename << "\" to write a webm";
93 return {};
94 }
95
96 if (!impl->segment_.Init(&impl->file_writer_)) {
97 LOG(ERROR) << "Failed to initialize the mkvkmuxer segment";
98 return {};
99 }
100
101 impl->segment_.AccurateClusterDuration(true);
102 impl->segment_.set_estimate_file_duration(true);
103
104 impl->encoder_factory_ = webrtc::CreateBuiltinVideoEncoderFactory();
105 if (!impl->encoder_factory_) {
106 LOG(ERROR) << "Failed to create webRTC built-in video encoder factory";
107 return {};
108 }
109
110 return std::unique_ptr<LocalRecorder>(new LocalRecorder(std::move(impl)));
111 }
112
LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)113 LocalRecorder::LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)
114 : impl_(std::move(impl)) {
115 }
116
117 LocalRecorder::~LocalRecorder() = default;
118
AddDisplay(size_t width,size_t height,std::shared_ptr<webrtc::VideoTrackSourceInterface> source)119 void LocalRecorder::AddDisplay(
120 size_t width,
121 size_t height,
122 std::shared_ptr<webrtc::VideoTrackSourceInterface> source) {
123 std::unique_ptr<Display> display(new Display(*impl_));
124 display->source_ = source;
125
126 display->video_encoder_ =
127 impl_->encoder_factory_->CreateVideoEncoder(webrtc::SdpVideoFormat("VP8"));
128 if (!display->video_encoder_) {
129 LOG(ERROR) << "Could not create vp8 video encoder";
130 return;
131 }
132 auto rc =
133 display->video_encoder_->RegisterEncodeCompleteCallback(display.get());
134 if (rc != 0) {
135 LOG(ERROR) << "Could not register encode complete callback";
136 return;
137 }
138 source->AddOrUpdateSink(display.get(), rtc::VideoSinkWants{});
139
140 webrtc::VideoCodec codec {};
141 memset(&codec, 0, sizeof(codec));
142 codec.codecType = webrtc::kVideoCodecVP8;
143 codec.width = width;
144 codec.height = height;
145 codec.startBitrate = 1000; // kilobits/sec
146 codec.maxBitrate = 2000;
147 codec.minBitrate = 0;
148 codec.maxFramerate = 60;
149 codec.active = true;
150 codec.qpMax = 56; // kDefaultMaxQp from simulcast_encoder_adapter.cc
151 codec.mode = webrtc::VideoCodecMode::kScreensharing;
152 codec.expect_encode_from_texture = false;
153 *codec.VP8() = webrtc::VideoEncoder::GetDefaultVp8Settings();
154
155 webrtc::VideoEncoder::Capabilities capabilities(false);
156 webrtc::VideoEncoder::Settings settings(capabilities, 1, 1 << 20);
157
158 rc = display->video_encoder_->InitEncode(&codec, settings);
159 if (rc != 0) {
160 LOG(ERROR) << "Failed to InitEncode";
161 return;
162 }
163
164 display->encoder_running_ = true;
165 display->encoder_thread_ = std::thread([](Display* display) {
166 display->EncoderLoop();
167 }, display.get());
168
169 std::lock_guard lock(impl_->mkv_mutex_);
170 display->video_track_number_ =
171 impl_->segment_.AddVideoTrack(width, height, 0);
172 if (display->video_track_number_ == 0) {
173 LOG(ERROR) << "Failed to add video track to webm muxer";
174 return;
175 }
176
177 impl_->displays_.emplace_back(std::move(display));
178 }
179
Stop()180 void LocalRecorder::Stop() {
181 for (auto& display : impl_->displays_) {
182 display->Stop();
183 }
184 impl_->displays_.clear();
185
186 std::lock_guard lock(impl_->mkv_mutex_);
187 impl_->segment_.Finalize();
188 }
189
Display(LocalRecorder::Impl & impl)190 LocalRecorder::Display::Display(LocalRecorder::Impl& impl) : impl_(impl) {
191 }
192
OnFrame(const webrtc::VideoFrame & frame)193 void LocalRecorder::Display::OnFrame(const webrtc::VideoFrame& frame) {
194 std::lock_guard queue_lock(encode_queue_mutex_);
195 static int kMaxQueuedFrames = 10;
196 if (encode_queue_.size() >= kMaxQueuedFrames) {
197 LOG(VERBOSE) << "Dropped frame, encoder queue too long";
198 return;
199 }
200 encode_queue_.push_back(frame);
201 encoder_queue_signal_.notify_one();
202 }
203
EncoderLoop()204 void LocalRecorder::Display::EncoderLoop() {
205 int frames_since_keyframe = 0;
206 std::chrono::time_point<std::chrono::steady_clock> start_timestamp;
207 auto last_keyframe_time = std::chrono::steady_clock::now();
208 while (encoder_running_) {
209 std::unique_ptr<webrtc::VideoFrame> frame;
210 {
211 std::unique_lock queue_lock(encode_queue_mutex_);
212 while (encode_queue_.size() == 0 && encoder_running_) {
213 encoder_queue_signal_.wait(queue_lock);
214 }
215 if (!encoder_running_) {
216 break;
217 }
218 frame = std::make_unique<webrtc::VideoFrame>(
219 std::move(encode_queue_.front()));
220 encode_queue_.pop_front();
221 }
222
223 auto now = std::chrono::steady_clock::now();
224 if (start_timestamp.time_since_epoch().count() == 0) {
225 start_timestamp = now;
226 }
227 auto timestamp_diff =
228 std::chrono::duration_cast<std::chrono::microseconds>(
229 now - start_timestamp);
230 frame->set_timestamp_us(timestamp_diff.count());
231 frame->set_timestamp(timestamp_diff.count() * kRtpTicksPerUs);
232
233 std::vector<webrtc::VideoFrameType> types;
234 auto time_since_keyframe = now - last_keyframe_time;
235 const auto min_keyframe_time = std::chrono::seconds(10);
236 if (frames_since_keyframe > 60 || time_since_keyframe > min_keyframe_time) {
237 last_keyframe_time = now;
238 frames_since_keyframe = 0;
239 types.push_back(webrtc::VideoFrameType::kVideoFrameKey);
240 } else {
241 types.push_back(webrtc::VideoFrameType::kVideoFrameDelta);
242 }
243 auto rc = video_encoder_->Encode(*frame, &types);
244 if (rc != 0) {
245 LOG(ERROR) << "Failed to encode frame";
246 }
247 }
248 }
249
Stop()250 void LocalRecorder::Display::Stop() {
251 encoder_running_ = false;
252 encoder_queue_signal_.notify_all();
253 if (encoder_thread_.joinable()) {
254 encoder_thread_.join();
255 }
256 }
257
OnEncodedImage(const webrtc::EncodedImage & encoded_image,const webrtc::CodecSpecificInfo * codec_specific_info,const webrtc::RTPFragmentationHeader * fragmentation)258 webrtc::EncodedImageCallback::Result LocalRecorder::Display::OnEncodedImage(
259 const webrtc::EncodedImage& encoded_image,
260 const webrtc::CodecSpecificInfo* codec_specific_info,
261 const webrtc::RTPFragmentationHeader* fragmentation) {
262 uint64_t timestamp = encoded_image.Timestamp() / kRtpTicksPerNs;
263
264 std::lock_guard(impl_.mkv_mutex_);
265
266 bool is_key =
267 encoded_image._frameType == webrtc::VideoFrameType::kVideoFrameKey;
268 bool success = impl_.segment_.AddFrame(
269 encoded_image.data(),
270 encoded_image.size(),
271 video_track_number_,
272 timestamp,
273 is_key);
274
275 webrtc::EncodedImageCallback::Result result(
276 success
277 ? webrtc::EncodedImageCallback::Result::Error::OK
278 : webrtc::EncodedImageCallback::Result::Error::ERROR_SEND_FAILED);
279
280 if (success) {
281 result.frame_id = encoded_image.Timestamp();
282 }
283 return result;
284 }
285
286 } // namespace webrtc_streaming
287 } // namespace cuttlefish
288
289