1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_frame_extractor.h"
17 #include "media_errors.h"
18 #include "media_log.h"
19 #include "scope_guard.h"
20 #include "time_perf.h"
21
22 namespace {
23 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaFrameExtract"};
24 }
25
26 namespace OHOS {
27 namespace Media {
28 static const std::unordered_map<int32_t, IPlayBinCtrler::PlayBinSeekMode> SEEK_OPTION_MAPPING = {
29 { AV_META_QUERY_NEXT_SYNC, IPlayBinCtrler::PlayBinSeekMode::NEXT_SYNC },
30 { AV_META_QUERY_PREVIOUS_SYNC, IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC },
31 { AV_META_QUERY_CLOSEST_SYNC, IPlayBinCtrler::PlayBinSeekMode::CLOSET_SYNC },
32 { AV_META_QUERY_CLOSEST, IPlayBinCtrler::PlayBinSeekMode::CLOSET },
33 };
34
AVMetaFrameExtractor()35 AVMetaFrameExtractor::AVMetaFrameExtractor()
36 {
37 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
38 }
39
~AVMetaFrameExtractor()40 AVMetaFrameExtractor::~AVMetaFrameExtractor()
41 {
42 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
43 Reset();
44 }
45
Init(const std::shared_ptr<IPlayBinCtrler> & playbin,GstElement & vidAppSink)46 int32_t AVMetaFrameExtractor::Init(const std::shared_ptr<IPlayBinCtrler> &playbin, GstElement &vidAppSink)
47 {
48 std::unique_lock<std::mutex> lock(mutex_);
49
50 if (playbin == nullptr) {
51 MEDIA_LOGE("playbin is nullptr");
52 return MSERR_INVALID_VAL;
53 }
54 playbin_ = playbin;
55 vidAppSink_ = GST_ELEMENT_CAST(gst_object_ref(&vidAppSink));
56
57 int32_t ret = SetupVideoSink();
58 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
59
60 return MSERR_OK;
61 }
62
ExtractFrame(int64_t timeUs,int32_t option,const OutputConfiguration & param)63 std::shared_ptr<AVSharedMemory> AVMetaFrameExtractor::ExtractFrame(
64 int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
65 {
66 int32_t ret = StartExtract(1, timeUs, option, param);
67 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "start extract failed");
68
69 auto outFrames = ExtractInternel();
70 StopExtract();
71
72 CHECK_AND_RETURN_RET_LOG(!outFrames.empty(), nullptr, "extract failed");
73 return outFrames[0];
74 }
75
ClearCache()76 void AVMetaFrameExtractor::ClearCache()
77 {
78 while (!originalFrames_.empty()) {
79 auto item = originalFrames_.front();
80 originalFrames_.pop();
81 gst_buffer_unref(item.first);
82 gst_caps_unref(item.second);
83 }
84 seekDone_ = false;
85 }
86
Reset()87 void AVMetaFrameExtractor::Reset()
88 {
89 std::unique_lock<std::mutex> lock(mutex_);
90
91 StopExtract();
92
93 decltype(signalIds_) tempSignalIds;
94 tempSignalIds.swap(signalIds_);
95
96 lock.unlock();
97 for(auto signalId : tempSignalIds) {
98 g_signal_handler_disconnect(vidAppSink_, signalId);
99 }
100 lock.lock();
101
102 if (vidAppSink_ != nullptr) {
103 gst_object_unref(vidAppSink_);
104 vidAppSink_ = nullptr;
105 }
106 playbin_ = nullptr;
107 }
108
ExtractInternel()109 std::vector<std::shared_ptr<AVSharedMemory>> AVMetaFrameExtractor::ExtractInternel()
110 {
111 std::vector<std::shared_ptr<AVSharedMemory>> outFrames;
112
113 std::unique_lock<std::mutex> lock(mutex_);
114 static constexpr int32_t timeout = 5;
115 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
116 return !originalFrames_.empty() || !startExtracting_;
117 });
118 CHECK_AND_RETURN_RET_LOG(startExtracting_, outFrames, "cancelled, exit frame extract");
119 CHECK_AND_RETURN_RET_LOG(!originalFrames_.empty(), outFrames, "no more frames");
120
121 auto item = originalFrames_.front();
122 originalFrames_.pop();
123 auto frameConverter = std::move(frameConverter_);
124 lock.unlock();
125
126 auto outFrame = frameConverter->Convert(*item.second, *item.first);
127 if (outFrame == nullptr) {
128 gst_buffer_unref(item.first);
129 gst_caps_unref(item.second);
130 MEDIA_LOGE("convert frame failed");
131 return outFrames;
132 }
133 gst_buffer_unref(item.first);
134 gst_caps_unref(item.second);
135
136 outFrames.push_back(outFrame);
137 MEDIA_LOGD("extract frame success, frame number: %{public}zu", outFrames.size());
138 return outFrames;
139 }
140
StartExtract(int32_t numFrames,int64_t timeUs,int32_t option,const OutputConfiguration & param)141 int32_t AVMetaFrameExtractor::StartExtract(
142 int32_t numFrames, int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
143 {
144 (void)numFrames;
145 std::unique_lock<std::mutex> lock(mutex_);
146
147 ON_SCOPE_EXIT(0) { StopExtract(); };
148
149 ClearCache();
150 startExtracting_ = true;
151
152 IPlayBinCtrler::PlayBinSeekMode mode = IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC;
153 if (SEEK_OPTION_MAPPING.find(option) != SEEK_OPTION_MAPPING.end()) {
154 mode = SEEK_OPTION_MAPPING.at(option);
155 }
156
157 int32_t ret = playbin_->Seek(timeUs, mode);
158 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "seek failed, cancel extract frames");
159
160 frameConverter_ = std::make_unique<AVMetaFrameConverter>();
161 ret = frameConverter_->Init(param);
162 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "init failed, cancel extract frames");
163
164 static constexpr int32_t timeout = 5;
165 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
166 return seekDone_ || !startExtracting_;
167 });
168 CHECK_AND_RETURN_RET(startExtracting_, MSERR_INVALID_OPERATION);
169
170 // no next sync frame, change to find the prev sync frame
171 if (originalFrames_.empty() && option == AVMetadataQueryOption::AV_META_QUERY_NEXT_SYNC) {
172 ClearCache();
173 mode = IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC;
174 ret = playbin_->Seek(timeUs, mode);
175 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "sek failed, cancel extract frames");
176 }
177
178 CANCEL_SCOPE_EXIT_GUARD(0);
179 return MSERR_OK;
180 }
181
StopExtract()182 void AVMetaFrameExtractor::StopExtract()
183 {
184 ClearCache();
185 startExtracting_ = false;
186 cond_.notify_all();
187 frameConverter_ = nullptr;
188 }
189
NotifyPlayBinMsg(const PlayBinMessage & msg)190 void AVMetaFrameExtractor::NotifyPlayBinMsg(const PlayBinMessage &msg)
191 {
192 std::unique_lock<std::mutex> lock(mutex_);
193
194 switch (msg.type) {
195 case PlayBinMsgType::PLAYBIN_MSG_SEEKDONE: {
196 seekDone_ = true;
197 cond_.notify_all();
198 break;
199 }
200 default:
201 break;
202 }
203 }
204
SetupVideoSink()205 int32_t AVMetaFrameExtractor::SetupVideoSink()
206 {
207 g_object_set(G_OBJECT(vidAppSink_), "emit-signals", TRUE, nullptr);
208 g_object_set(G_OBJECT(vidAppSink_), "max-buffers", 1, nullptr);
209
210 gulong signalId = g_signal_connect(G_OBJECT(vidAppSink_), "new-preroll", G_CALLBACK(OnNewPrerollArrived), this);
211 CHECK_AND_RETURN_RET_LOG(signalId != 0, MSERR_INVALID_OPERATION, "listen to new-preroll failed");
212 signalIds_.push_back(signalId);
213
214 return MSERR_OK;
215 }
216
OnNewPrerollArrived(GstElement * sink,AVMetaFrameExtractor * thiz)217 GstFlowReturn AVMetaFrameExtractor::OnNewPrerollArrived(GstElement *sink, AVMetaFrameExtractor *thiz)
218 {
219 CHECK_AND_RETURN_RET(thiz != nullptr, GST_FLOW_ERROR);
220 CHECK_AND_RETURN_RET(sink != nullptr, GST_FLOW_ERROR);
221
222 std::unique_lock<std::mutex> lock(thiz->mutex_);
223
224 GstSample *sample = nullptr;
225 g_signal_emit_by_name(sink, "pull-preroll", &sample);
226 CHECK_AND_RETURN_RET(sample != nullptr, GST_FLOW_ERROR);
227
228 ON_SCOPE_EXIT(0) { gst_sample_unref(sample); };
229
230 GstBuffer *buffer = gst_sample_get_buffer(sample);
231 CHECK_AND_RETURN_RET(buffer != nullptr, GST_FLOW_ERROR);
232 MEDIA_LOGI("preroll buffer arrived, pts: %{public}" PRIu64 "", GST_BUFFER_PTS(buffer));
233
234 if (!thiz->startExtracting_) {
235 MEDIA_LOGI("not start extract, ignore");
236 return GST_FLOW_OK;
237 }
238
239 GstCaps *caps = gst_sample_get_caps(sample);
240 CHECK_AND_RETURN_RET(caps != nullptr, GST_FLOW_ERROR);
241 thiz->originalFrames_.push({ gst_buffer_ref(buffer), gst_caps_ref(caps) });
242
243 thiz->cond_.notify_all();
244 return GST_FLOW_OK;
245 }
246 } // namespace Media
247 } // namespace OHOS
248