1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/frontend/webrtc/libdevice/local_recorder.h"
18
19 #include <atomic>
20 #include <chrono>
21 #include <list>
22 #include <mutex>
23 #include <thread>
24 #include <vector>
25
26 #include <android-base/logging.h>
27 #include <api/media_stream_interface.h>
28 #include <api/rtp_parameters.h>
29 #include <api/task_queue/default_task_queue_factory.h>
30 #include <api/video/builtin_video_bitrate_allocator_factory.h>
31 #include <api/video_codecs/builtin_video_encoder_factory.h>
32 #include <api/video_codecs/video_encoder.h>
33 #include <mkvmuxer/mkvmuxer.h>
34 #include <mkvmuxer/mkvwriter.h>
35 #include <system_wrappers/include/clock.h>
36
37 namespace cuttlefish {
38 namespace webrtc_streaming {
39
40 constexpr double kRtpTicksPerSecond = 90000.;
41 constexpr double kRtpTicksPerMs = kRtpTicksPerSecond / 1000.;
42 constexpr double kRtpTicksPerUs = kRtpTicksPerMs / 1000.;
43 constexpr double kRtpTicksPerNs = kRtpTicksPerUs / 1000.;
44
45 class LocalRecorder::Display
46 : public webrtc::EncodedImageCallback
47 , public rtc::VideoSinkInterface<webrtc::VideoFrame> {
48 public:
49 Display(LocalRecorder::Impl& impl);
50
51 void EncoderLoop();
52 void Stop();
53
54 // VideoSinkInterface
55 virtual void OnFrame(const webrtc::VideoFrame& frame) override;
56
57 // EncodedImageCallback
58 virtual webrtc::EncodedImageCallback::Result OnEncodedImage(
59 const webrtc::EncodedImage& encoded_image,
60 const webrtc::CodecSpecificInfo* codec_specific_info) override;
61
62 LocalRecorder::Impl& impl_;
63 std::shared_ptr<webrtc::VideoTrackSourceInterface> source_;
64 std::unique_ptr<webrtc::VideoEncoder> video_encoder_;
65 uint64_t video_track_number_;
66
67 // TODO(schuffelen): Use a WebRTC task queue?
68 std::thread encoder_thread_;
69 std::condition_variable encoder_queue_signal_;
70 std::mutex encode_queue_mutex_;
71 std::list<webrtc::VideoFrame> encode_queue_;
72 std::atomic_bool encoder_running_ = true;
73 };
74
75 class LocalRecorder::Impl {
76 public:
77 mkvmuxer::MkvWriter file_writer_;
78 mkvmuxer::Segment segment_;
79 std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
80 std::mutex mkv_mutex_;
81 std::vector<std::unique_ptr<Display>> displays_;
82 };
83
84 /* static */
Create(const std::string & filename)85 std::unique_ptr<LocalRecorder> LocalRecorder::Create(
86 const std::string& filename) {
87 std::unique_ptr<Impl> impl(new Impl());
88
89 if (!impl->file_writer_.Open(filename.c_str())) {
90 LOG(ERROR) << "Failed to open \"" << filename << "\" to write a webm";
91 return {};
92 }
93
94 if (!impl->segment_.Init(&impl->file_writer_)) {
95 LOG(ERROR) << "Failed to initialize the mkvkmuxer segment";
96 return {};
97 }
98
99 impl->segment_.AccurateClusterDuration(true);
100 impl->segment_.set_estimate_file_duration(true);
101
102 impl->encoder_factory_ = webrtc::CreateBuiltinVideoEncoderFactory();
103 if (!impl->encoder_factory_) {
104 LOG(ERROR) << "Failed to create webRTC built-in video encoder factory";
105 return {};
106 }
107
108 return std::unique_ptr<LocalRecorder>(new LocalRecorder(std::move(impl)));
109 }
110
LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)111 LocalRecorder::LocalRecorder(std::unique_ptr<LocalRecorder::Impl> impl)
112 : impl_(std::move(impl)) {
113 }
114
115 LocalRecorder::~LocalRecorder() = default;
116
AddDisplay(size_t width,size_t height,std::shared_ptr<webrtc::VideoTrackSourceInterface> source)117 void LocalRecorder::AddDisplay(
118 size_t width,
119 size_t height,
120 std::shared_ptr<webrtc::VideoTrackSourceInterface> source) {
121 std::unique_ptr<Display> display(new Display(*impl_));
122 display->source_ = source;
123
124 std::lock_guard lock(impl_->mkv_mutex_);
125 display->video_track_number_ =
126 impl_->segment_.AddVideoTrack(width, height, 0);
127 if (display->video_track_number_ == 0) {
128 LOG(ERROR) << "Failed to add video track to webm muxer";
129 return;
130 }
131
132 display->video_encoder_ =
133 impl_->encoder_factory_->CreateVideoEncoder(webrtc::SdpVideoFormat("VP8"));
134 if (!display->video_encoder_) {
135 LOG(ERROR) << "Could not create vp8 video encoder";
136 return;
137 }
138 auto rc =
139 display->video_encoder_->RegisterEncodeCompleteCallback(display.get());
140 if (rc != 0) {
141 LOG(ERROR) << "Could not register encode complete callback";
142 return;
143 }
144 source->AddOrUpdateSink(display.get(), rtc::VideoSinkWants{});
145
146 webrtc::VideoCodec codec {};
147 memset(&codec, 0, sizeof(codec));
148 codec.codecType = webrtc::kVideoCodecVP8;
149 codec.width = width;
150 codec.height = height;
151 codec.startBitrate = 1000; // kilobits/sec
152 codec.maxBitrate = 2000;
153 codec.minBitrate = 0;
154 codec.maxFramerate = 60;
155 codec.active = true;
156 codec.qpMax = 56; // kDefaultMaxQp from simulcast_encoder_adapter.cc
157 codec.mode = webrtc::VideoCodecMode::kScreensharing;
158 codec.expect_encode_from_texture = false;
159 *codec.VP8() = webrtc::VideoEncoder::GetDefaultVp8Settings();
160
161 webrtc::VideoEncoder::Capabilities capabilities(false);
162 webrtc::VideoEncoder::Settings settings(capabilities, 1, 1 << 20);
163
164 rc = display->video_encoder_->InitEncode(&codec, settings);
165 if (rc != 0) {
166 LOG(ERROR) << "Failed to InitEncode";
167 return;
168 }
169
170 display->encoder_running_ = true;
171 display->encoder_thread_ = std::thread([](Display* display) {
172 display->EncoderLoop();
173 }, display.get());
174
175 impl_->displays_.emplace_back(std::move(display));
176 }
177
Stop()178 void LocalRecorder::Stop() {
179 for (auto& display : impl_->displays_) {
180 display->Stop();
181 }
182 impl_->displays_.clear();
183
184 std::lock_guard lock(impl_->mkv_mutex_);
185 impl_->segment_.Finalize();
186 }
187
Display(LocalRecorder::Impl & impl)188 LocalRecorder::Display::Display(LocalRecorder::Impl& impl) : impl_(impl) {
189 }
190
OnFrame(const webrtc::VideoFrame & frame)191 void LocalRecorder::Display::OnFrame(const webrtc::VideoFrame& frame) {
192 std::lock_guard queue_lock(encode_queue_mutex_);
193 static int kMaxQueuedFrames = 10;
194 if (encode_queue_.size() >= kMaxQueuedFrames) {
195 LOG(VERBOSE) << "Dropped frame, encoder queue too long";
196 return;
197 }
198 encode_queue_.push_back(frame);
199 encoder_queue_signal_.notify_one();
200 }
201
EncoderLoop()202 void LocalRecorder::Display::EncoderLoop() {
203 int frames_since_keyframe = 0;
204 std::chrono::time_point<std::chrono::steady_clock> start_timestamp;
205 auto last_keyframe_time = std::chrono::steady_clock::now();
206 while (encoder_running_) {
207 std::unique_ptr<webrtc::VideoFrame> frame;
208 {
209 std::unique_lock queue_lock(encode_queue_mutex_);
210 while (encode_queue_.size() == 0 && encoder_running_) {
211 encoder_queue_signal_.wait(queue_lock);
212 }
213 if (!encoder_running_) {
214 break;
215 }
216 frame = std::make_unique<webrtc::VideoFrame>(
217 std::move(encode_queue_.front()));
218 encode_queue_.pop_front();
219 }
220
221 auto now = std::chrono::steady_clock::now();
222 if (start_timestamp.time_since_epoch().count() == 0) {
223 start_timestamp = now;
224 }
225 auto timestamp_diff =
226 std::chrono::duration_cast<std::chrono::microseconds>(
227 now - start_timestamp);
228 frame->set_timestamp_us(timestamp_diff.count());
229 frame->set_timestamp(timestamp_diff.count() * kRtpTicksPerUs);
230
231 std::vector<webrtc::VideoFrameType> types;
232 auto time_since_keyframe = now - last_keyframe_time;
233 const auto min_keyframe_time = std::chrono::seconds(10);
234 if (frames_since_keyframe > 60 || time_since_keyframe > min_keyframe_time) {
235 last_keyframe_time = now;
236 frames_since_keyframe = 0;
237 types.push_back(webrtc::VideoFrameType::kVideoFrameKey);
238 } else {
239 types.push_back(webrtc::VideoFrameType::kVideoFrameDelta);
240 }
241 auto rc = video_encoder_->Encode(*frame, &types);
242 if (rc != 0) {
243 LOG(ERROR) << "Failed to encode frame";
244 }
245 }
246 }
247
Stop()248 void LocalRecorder::Display::Stop() {
249 encoder_running_ = false;
250 encoder_queue_signal_.notify_all();
251 if (encoder_thread_.joinable()) {
252 encoder_thread_.join();
253 }
254 }
255
OnEncodedImage(const webrtc::EncodedImage & encoded_image,const webrtc::CodecSpecificInfo * codec_specific_info)256 webrtc::EncodedImageCallback::Result LocalRecorder::Display::OnEncodedImage(
257 const webrtc::EncodedImage& encoded_image,
258 const webrtc::CodecSpecificInfo* codec_specific_info) {
259 uint64_t timestamp = encoded_image.Timestamp() / kRtpTicksPerNs;
260
261 std::lock_guard(impl_.mkv_mutex_);
262
263 bool is_key =
264 encoded_image._frameType == webrtc::VideoFrameType::kVideoFrameKey;
265 bool success = impl_.segment_.AddFrame(
266 encoded_image.data(),
267 encoded_image.size(),
268 video_track_number_,
269 timestamp,
270 is_key);
271
272 webrtc::EncodedImageCallback::Result result(
273 success
274 ? webrtc::EncodedImageCallback::Result::Error::OK
275 : webrtc::EncodedImageCallback::Result::Error::ERROR_SEND_FAILED);
276
277 if (success) {
278 result.frame_id = encoded_image.Timestamp();
279 }
280 return result;
281 }
282
283 } // namespace webrtc_streaming
284 } // namespace cuttlefish
285
286