1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "avmeta_frame_extractor.h"
17 #include "media_errors.h"
18 #include "media_log.h"
19 #include "scope_guard.h"
20
21 namespace {
22 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaFrameExtract"};
23 }
24
25 namespace OHOS {
26 namespace Media {
27 static const std::unordered_map<int32_t, IPlayBinCtrler::PlayBinSeekMode> SEEK_OPTION_MAPPING = {
28 { AV_META_QUERY_NEXT_SYNC, IPlayBinCtrler::PlayBinSeekMode::NEXT_SYNC },
29 { AV_META_QUERY_PREVIOUS_SYNC, IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC },
30 { AV_META_QUERY_CLOSEST_SYNC, IPlayBinCtrler::PlayBinSeekMode::CLOSET_SYNC },
31 { AV_META_QUERY_CLOSEST, IPlayBinCtrler::PlayBinSeekMode::CLOSET },
32 };
33
AVMetaFrameExtractor()34 AVMetaFrameExtractor::AVMetaFrameExtractor()
35 {
36 MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
37 }
38
~AVMetaFrameExtractor()39 AVMetaFrameExtractor::~AVMetaFrameExtractor()
40 {
41 MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this));
42 Reset();
43 }
44
Init(const std::shared_ptr<IPlayBinCtrler> & playbin,GstElement & vidAppSink)45 int32_t AVMetaFrameExtractor::Init(const std::shared_ptr<IPlayBinCtrler> &playbin, GstElement &vidAppSink)
46 {
47 std::unique_lock<std::mutex> lock(mutex_);
48
49 CHECK_AND_RETURN_RET_LOG(playbin != nullptr, MSERR_INVALID_VAL, "playbin is nullptr");
50 playbin_ = playbin;
51 vidAppSink_ = GST_ELEMENT_CAST(gst_object_ref(&vidAppSink));
52
53 int32_t ret = SetupVideoSink();
54 CHECK_AND_RETURN_RET(ret == MSERR_OK, ret);
55
56 return MSERR_OK;
57 }
58
ExtractFrame(int64_t timeUs,int32_t option,const OutputConfiguration & param)59 std::shared_ptr<AVSharedMemory> AVMetaFrameExtractor::ExtractFrame(
60 int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
61 {
62 int32_t ret = StartExtract(1, timeUs, option, param);
63 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, nullptr, "start extract failed");
64
65 auto outFrames = ExtractInternel();
66 StopExtract();
67
68 CHECK_AND_RETURN_RET_LOG(!outFrames.empty(), nullptr, "extract failed");
69 return outFrames[0];
70 }
71
ClearCache()72 void AVMetaFrameExtractor::ClearCache()
73 {
74 while (!originalFrames_.empty()) {
75 auto item = originalFrames_.front();
76 originalFrames_.pop();
77 gst_buffer_unref(item.first);
78 gst_caps_unref(item.second);
79 }
80 seekDone_ = false;
81 }
82
Reset()83 void AVMetaFrameExtractor::Reset()
84 {
85 std::unique_lock<std::mutex> lock(mutex_);
86
87 StopExtract();
88
89 decltype(signalIds_) tempSignalIds;
90 tempSignalIds.swap(signalIds_);
91
92 lock.unlock();
93 for(auto signalId : tempSignalIds) {
94 g_signal_handler_disconnect(vidAppSink_, signalId);
95 }
96 lock.lock();
97
98 if (vidAppSink_ != nullptr) {
99 gst_object_unref(vidAppSink_);
100 vidAppSink_ = nullptr;
101 }
102 playbin_ = nullptr;
103 }
104
ExtractInternel()105 std::vector<std::shared_ptr<AVSharedMemory>> AVMetaFrameExtractor::ExtractInternel()
106 {
107 std::vector<std::shared_ptr<AVSharedMemory>> outFrames;
108
109 std::unique_lock<std::mutex> lock(mutex_);
110 static constexpr int32_t timeout = 3;
111 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
112 return !originalFrames_.empty() || !startExtracting_;
113 });
114 CHECK_AND_RETURN_RET_LOG(startExtracting_, outFrames, "cancelled, exit frame extract");
115 CHECK_AND_RETURN_RET_LOG(!originalFrames_.empty(), outFrames, "no more frames");
116
117 auto item = originalFrames_.front();
118 originalFrames_.pop();
119 auto frameConverter = std::move(frameConverter_);
120 lock.unlock();
121
122 auto outFrame = frameConverter->Convert(*item.second, *item.first);
123 if (outFrame == nullptr) {
124 gst_buffer_unref(item.first);
125 gst_caps_unref(item.second);
126 MEDIA_LOGE("convert frame failed");
127 return outFrames;
128 }
129 gst_buffer_unref(item.first);
130 gst_caps_unref(item.second);
131
132 outFrames.push_back(outFrame);
133 MEDIA_LOGD("extract frame success, frame number: %{public}zu", outFrames.size());
134 return outFrames;
135 }
136
StartExtract(int32_t numFrames,int64_t timeUs,int32_t option,const OutputConfiguration & param)137 int32_t AVMetaFrameExtractor::StartExtract(
138 int32_t numFrames, int64_t timeUs, int32_t option, const OutputConfiguration ¶m)
139 {
140 (void)numFrames;
141 std::unique_lock<std::mutex> lock(mutex_);
142
143 ON_SCOPE_EXIT(0) { StopExtract(); };
144
145 ClearCache();
146 startExtracting_ = true;
147
148 IPlayBinCtrler::PlayBinSeekMode mode = IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC;
149 if (SEEK_OPTION_MAPPING.find(option) != SEEK_OPTION_MAPPING.end()) {
150 mode = SEEK_OPTION_MAPPING.at(option);
151 }
152
153 int32_t ret = playbin_->Seek(timeUs, mode);
154 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "seek failed, cancel extract frames");
155
156 frameConverter_ = std::make_unique<AVMetaFrameConverter>();
157 ret = frameConverter_->Init(param);
158 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "init failed, cancel extract frames");
159
160 static constexpr int32_t timeout = 3;
161 cond_.wait_for(lock, std::chrono::seconds(timeout), [this]() {
162 return seekDone_ || !startExtracting_;
163 });
164 CHECK_AND_RETURN_RET(startExtracting_, MSERR_INVALID_OPERATION);
165
166 // no next sync frame, change to find the prev sync frame
167 if (originalFrames_.empty() && option == AVMetadataQueryOption::AV_META_QUERY_NEXT_SYNC) {
168 ClearCache();
169 mode = IPlayBinCtrler::PlayBinSeekMode::PREV_SYNC;
170 ret = playbin_->Seek(timeUs, mode);
171 CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, ret, "sek failed, cancel extract frames");
172 }
173
174 CANCEL_SCOPE_EXIT_GUARD(0);
175 return MSERR_OK;
176 }
177
StopExtract()178 void AVMetaFrameExtractor::StopExtract()
179 {
180 ClearCache();
181 startExtracting_ = false;
182 cond_.notify_all();
183 frameConverter_ = nullptr;
184 }
185
NotifyPlayBinMsg(const PlayBinMessage & msg)186 void AVMetaFrameExtractor::NotifyPlayBinMsg(const PlayBinMessage &msg)
187 {
188 std::unique_lock<std::mutex> lock(mutex_);
189
190 switch (msg.type) {
191 case PlayBinMsgType::PLAYBIN_MSG_SEEKDONE: {
192 seekDone_ = true;
193 cond_.notify_all();
194 break;
195 }
196 default:
197 break;
198 }
199 }
200
SetupVideoSink()201 int32_t AVMetaFrameExtractor::SetupVideoSink()
202 {
203 g_object_set(G_OBJECT(vidAppSink_), "emit-signals", TRUE, nullptr);
204 g_object_set(G_OBJECT(vidAppSink_), "max-buffers", 1, nullptr);
205
206 gulong signalId = g_signal_connect(G_OBJECT(vidAppSink_), "new-preroll", G_CALLBACK(OnNewPrerollArrived), this);
207 CHECK_AND_RETURN_RET_LOG(signalId != 0, MSERR_INVALID_OPERATION, "listen to new-preroll failed");
208 signalIds_.push_back(signalId);
209
210 return MSERR_OK;
211 }
212
OnNewPrerollArrived(GstElement * sink,AVMetaFrameExtractor * thiz)213 GstFlowReturn AVMetaFrameExtractor::OnNewPrerollArrived(GstElement *sink, AVMetaFrameExtractor *thiz)
214 {
215 CHECK_AND_RETURN_RET(thiz != nullptr, GST_FLOW_ERROR);
216 CHECK_AND_RETURN_RET(sink != nullptr, GST_FLOW_ERROR);
217
218 std::unique_lock<std::mutex> lock(thiz->mutex_);
219
220 GstSample *sample = nullptr;
221 g_signal_emit_by_name(sink, "pull-preroll", &sample);
222 CHECK_AND_RETURN_RET(sample != nullptr, GST_FLOW_ERROR);
223
224 ON_SCOPE_EXIT(0) { gst_sample_unref(sample); };
225
226 GstBuffer *buffer = gst_sample_get_buffer(sample);
227 CHECK_AND_RETURN_RET(buffer != nullptr, GST_FLOW_ERROR);
228 MEDIA_LOGI("preroll buffer arrived, pts: %{public}" PRIu64 "", GST_BUFFER_PTS(buffer));
229
230 if (!thiz->startExtracting_) {
231 MEDIA_LOGI("not start extract, ignore");
232 return GST_FLOW_OK;
233 }
234
235 GstCaps *caps = gst_sample_get_caps(sample);
236 CHECK_AND_RETURN_RET(caps != nullptr, GST_FLOW_ERROR);
237 thiz->originalFrames_.push({ gst_buffer_ref(buffer), gst_caps_ref(caps) });
238
239 thiz->cond_.notify_all();
240 return GST_FLOW_OK;
241 }
242 } // namespace Media
243 } // namespace OHOS
244