• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef SCREEN_CAPTURE_SERVICE_SERVER_BASE_H
17 #define SCREEN_CAPTURE_SERVICE_SERVER_BASE_H
18 
19 #include <mutex>
20 #include <cstdlib>
21 #include <thread>
22 #include <string>
23 #include <memory>
24 #include <atomic>
25 #include <queue>
26 #include <vector>
27 #include <chrono>
28 
29 #include "audio_capturer_wrapper.h"
30 #include "i_screen_capture_service.h"
31 #include "nocopyable.h"
32 #include "uri_helper.h"
33 #include "task_queue.h"
34 #include "accesstoken_kit.h"
35 #include "privacy_kit.h"
36 #include "ipc_skeleton.h"
37 #include "screen_capture.h"
38 #include "audio_capturer.h"
39 #include "nativetoken_kit.h"
40 #include "token_setproc.h"
41 #include "audio_info.h"
42 #include "surface.h"
43 #include "display_manager.h"
44 #include "screen_manager.h"
45 #include "i_recorder_service.h"
46 #include "recorder_server.h"
47 #include "notification_content.h"
48 #include "notification_helper.h"
49 #include "notification_request.h"
50 #include "notification_constant.h"
51 #include "notification_slot.h"
52 #ifdef SUPPORT_CALL
53 #include "incall_observer.h"
54 #endif
55 #include "account_observer.h"
56 #include "media_data_source.h"
57 #include "meta/meta.h"
58 #include "audio_stream_manager.h"
59 #include "screen_capture_monitor_server.h"
60 #include "json/json.h"
61 #include "tokenid_kit.h"
62 #include "window_manager.h"
63 #include "limitIdGenerator.h"
64 #include "system_ability_status_change_stub.h"
65 #include "i_input_device_listener.h"
66 #include "input_manager.h"
67 #include "session_lifecycle_listener_stub.h"
68 #include "common_event_manager.h"
69 
70 namespace OHOS {
71 namespace Media {
72 using namespace Rosen;
73 using namespace AudioStandard;
74 using namespace OHOS::Notification;
75 using OHOS::Security::AccessToken::PrivacyKit;
76 
77 class ScreenCaptureServer;
78 
79 class NotificationSubscriber : public OHOS::Notification::NotificationLocalLiveViewSubscriber {
80 public:
81     void OnConnected() override;
82     void OnDisconnected() override;
83     void OnResponse(int32_t notificationId,
84         OHOS::sptr<OHOS::Notification::NotificationButtonOption> buttonOption) override;
85     void OnDied() override;
86 };
87 
88 enum VideoPermissionState : int32_t {
89     START_VIDEO = 0,
90     STOP_VIDEO = 1
91 };
92 
93 enum AVScreenCaptureState : int32_t {
94     CREATED = 0,
95     POPUP_WINDOW = 1,
96     STARTING = 2,
97     STARTED = 3,
98     STOPPED = 4
99 };
100 
101 enum AVScreenCaptureMixMode : int32_t {
102     MIX_MODE = 0,
103     MIC_MODE = 1,
104     INNER_MODE = 2,
105     INVALID_MODE = 3
106 };
107 
108 enum class AVScreenCaptureMixBufferType : int32_t {
109     MIX = 0,
110     MIC = 1,
111     INNER = 2,
112     INVALID = 3
113 };
114 
115 enum AVScreenCaptureAvType : int8_t {
116     INVALID_TYPE = -1,
117     AUDIO_TYPE = 0,
118     VIDEO_TYPE = 1,
119     AV_TYPE = 2
120 };
121 
122 enum AVScreenCaptureDataMode : int8_t {
123     BUFFER_MODE = 0,
124     SUFFACE_MODE = 1,
125     FILE_MODE = 2
126 };
127 
128 enum StopReason: int8_t {
129     NORMAL_STOPPED = 0,
130     RECEIVE_USER_PRIVACY_AUTHORITY_FAILED = 1,
131     POST_START_SCREENCAPTURE_HANDLE_FAILURE = 2,
132     REQUEST_USER_PRIVACY_AUTHORITY_FAILED = 3,
133     STOP_REASON_INVALID = 4
134 };
135 
136 struct SurfaceBufferEntry {
SurfaceBufferEntrySurfaceBufferEntry137     SurfaceBufferEntry(sptr<OHOS::SurfaceBuffer> buf, int32_t fence, int64_t timeStamp, OHOS::Rect& damage)
138         : buffer(std::move(buf)), flushFence(fence), timeStamp(timeStamp), damageRect(damage) {}
139     ~SurfaceBufferEntry() noexcept = default;
140 
141     sptr<OHOS::SurfaceBuffer> buffer;
142     int32_t flushFence;
143     int64_t timeStamp = 0;
144     OHOS::Rect damageRect = {0, 0, 0, 0};
145 };
146 
147 struct StatisticalEventInfo {
148     int32_t errCode = 0;
149     std::string errMsg;
150     int32_t captureDuration = -1;
151     bool userAgree = false;
152     bool requireMic = false;
153     bool enableMic = false;
154     std::string videoResolution;
155     StopReason stopReason = StopReason::STOP_REASON_INVALID;
156     int32_t startLatency = -1;
157 };
158 
159 enum class SCBufferMessageType {
160     EXIT,
161     GET_BUFFER
162 };
163 
164 struct SCBufferMessage {
165     SCBufferMessageType type;
166     std::string text;
167 };
168 
169 class ScreenCapBufferConsumerListener : public IBufferConsumerListener {
170 public:
ScreenCapBufferConsumerListener(sptr<Surface> consumer,const std::shared_ptr<ScreenCaptureCallBack> & screenCaptureCb)171     ScreenCapBufferConsumerListener(
172         sptr<Surface> consumer, const std::shared_ptr<ScreenCaptureCallBack> &screenCaptureCb)
173         : consumer_(consumer), screenCaptureCb_(screenCaptureCb) {}
174     ~ScreenCapBufferConsumerListener();
175 
176     void OnBufferAvailable() override;
177     int32_t AcquireVideoBuffer(sptr<OHOS::SurfaceBuffer> &surfaceBuffer, int32_t &fence, int64_t &timestamp,
178         OHOS::Rect &damage);
179     int32_t ReleaseVideoBuffer();
180     int32_t Release();
181     int32_t StartBufferThread();
182     void OnBufferAvailableAction();
183     void SurfaceBufferThreadRun();
184     void StopBufferThread();
185 
186 private:
187     int32_t ReleaseBuffer();
188     void ProcessVideoBufferCallBack();
189 
190 private:
191     std::mutex bufferAvailableWorkerMtx_;
192     std::condition_variable bufferAvailableWorkerCv_;
193     std::queue<SCBufferMessage> messageQueueSCB_;
194 
195     std::mutex mutex_;
196     sptr<OHOS::Surface> consumer_ = nullptr;
197     std::shared_ptr<ScreenCaptureCallBack> screenCaptureCb_ = nullptr;
198     std::atomic<bool> isSurfaceCbInThreadStopped_ {true};
199     std::thread* surfaceCbInThread_ = nullptr;
200 
201     std::mutex bufferMutex_;
202     std::condition_variable bufferCond_;
203     std::queue<std::unique_ptr<SurfaceBufferEntry>> availBuffers_;
204 
205     static constexpr uint64_t MAX_MESSAGE_QUEUE_SIZE = 5;
206     static constexpr uint32_t MAX_BUFFER_SIZE = 3;
207     static constexpr uint32_t OPERATION_TIMEOUT_IN_MS = 1000; // 1000ms
208 };
209 
210 #ifdef SUPPORT_CALL
211 class ScreenCaptureObserverCallBack : public InCallObserverCallBack, public AccountObserverCallBack {
212 #else
213 class ScreenCaptureObserverCallBack : public AccountObserverCallBack {
214 #endif
215 public:
216     explicit ScreenCaptureObserverCallBack(std::weak_ptr<ScreenCaptureServer> screenCaptureServer);
217     ~ScreenCaptureObserverCallBack();
218     bool StopAndRelease(AVScreenCaptureStateCode state) override;
219     bool NotifyStopAndRelease(AVScreenCaptureStateCode state) override;
220 #ifdef SUPPORT_CALL
221     bool TelCallStateUpdated(bool isInCall) override;
222     bool NotifyTelCallStateUpdated(bool isInCall) override;
223 #endif
224     void Release() override;
225 private:
226     std::weak_ptr<ScreenCaptureServer> screenCaptureServer_;
227     TaskQueue taskQueObserverCb_;
228     std::mutex mutex_;
229 };
230 
231 class AudioDataSource : public IAudioDataSource {
232 public:
AudioDataSource(AVScreenCaptureMixMode type,ScreenCaptureServer * screenCaptureServer)233     AudioDataSource(AVScreenCaptureMixMode type, ScreenCaptureServer* screenCaptureServer) : type_(type),
234         screenCaptureServer_(screenCaptureServer) {}
235 
236     int64_t GetFirstAudioTime(std::shared_ptr<AudioBuffer> &innerAudioBuffer,
237         std::shared_ptr<AudioBuffer> &micAudioBuffer);
238     AudioDataSourceReadAtActionState WriteInnerAudio(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
239         std::shared_ptr<AudioBuffer> &innerAudioBuffer);
240     AudioDataSourceReadAtActionState WriteMicAudio(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
241         std::shared_ptr<AudioBuffer> &micAudioBuffer);
242     AudioDataSourceReadAtActionState WriteMixAudio(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
243         std::shared_ptr<AudioBuffer> &innerAudioBuffer, std::shared_ptr<AudioBuffer> &micAudioBuffer);
244     AudioDataSourceReadAtActionState ReadWriteAudioBufferMix(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
245         std::shared_ptr<AudioBuffer> &innerAudioBuffer, std::shared_ptr<AudioBuffer> &micAudioBuffer);
246     AudioDataSourceReadAtActionState ReadWriteAudioBufferMixCore(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
247         std::shared_ptr<AudioBuffer> &innerAudioBuffer, std::shared_ptr<AudioBuffer> &micAudioBuffer);
248     AudioDataSourceReadAtActionState HandleMicBeforeInnerSync(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
249         std::shared_ptr<AudioBuffer> &innerAudioBuffer, std::shared_ptr<AudioBuffer> &micAudioBuffer);
250     AudioDataSourceReadAtActionState InnerMicAudioSync(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
251         std::shared_ptr<AudioBuffer> &innerAudioBuffer, std::shared_ptr<AudioBuffer> &micAudioBuffer);
252     AudioDataSourceReadAtActionState VideoAudioSyncMixMode(std::shared_ptr<AVBuffer> &buffer, uint32_t length,
253         int64_t timeWindow, std::shared_ptr<AudioBuffer> &innerAudioBuffer,
254         std::shared_ptr<AudioBuffer> &micAudioBuffer);
255     AudioDataSourceReadAtActionState ReadAtMixMode(std::shared_ptr<AVBuffer> buffer, uint32_t length);
256     AudioDataSourceReadAtActionState ReadAtMicMode(std::shared_ptr<AVBuffer> buffer, uint32_t length);
257     AudioDataSourceReadAtActionState ReadAtInnerMode(std::shared_ptr<AVBuffer> buffer, uint32_t length);
258     AudioDataSourceReadAtActionState ReadAt(std::shared_ptr<AVBuffer> buffer, uint32_t length) override;
259     AudioDataSourceReadAtActionState VideoAudioSyncInnerMode(std::shared_ptr<AVBuffer> &buffer,
260         uint32_t length, int64_t timeWindow, std::shared_ptr<AudioBuffer> &innerAudioBuffer);
261     int32_t GetSize(int64_t &size) override;
262     int32_t RegisterAudioRendererEventListener(const int32_t clientPid,
263         const std::shared_ptr<AudioRendererStateChangeCallback> &callback);
264     int32_t UnregisterAudioRendererEventListener(const int32_t clientPid);
265     void SpeakerStateUpdate(
266         const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
267     bool HasSpeakerStream(
268         const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
269     void VoIPStateUpdate(
270         const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
271     bool HasVoIPStream(
272         const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
273     void SetAppPid(int32_t appid);
274     void SetAppName(std::string appName);
275     int32_t GetAppPid();
276     bool GetIsInVoIPCall();
277     bool GetSpeakerAliveStatus();
278     bool IsInWaitMicSyncState();
279 #ifdef SUPPORT_CALL
280     void TelCallAudioStateUpdate(
281         const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
282 #endif
283     void SetVideoFirstFramePts(int64_t firstFramePts) override;
284     void SetAudioFirstFramePts(int64_t firstFramePts);
285     AudioDataSourceReadAtActionState MixModeBufferWrite(std::shared_ptr<AudioBuffer> &innerAudioBuffer,
286         std::shared_ptr<AudioBuffer> &micAudioBuffer, std::shared_ptr<AVMemory> &bufferMem);
287     void HandlePastMicBuffer(std::shared_ptr<AudioBuffer> &micAudioBuffer);
288     void HandleSwitchToSpeakerOptimise(std::shared_ptr<AudioBuffer> &innerAudioBuffer,
289         std::shared_ptr<AudioBuffer> &micAudioBuffer);
290     void HandleBufferTimeStamp(std::shared_ptr<AudioBuffer> &innerAudioBuffer,
291         std::shared_ptr<AudioBuffer> &micAudioBuffer);
292     ScreenCaptureServer* GetScreenCaptureServer();
293 private:
294     void MixAudio(char** srcData, char* mixData, int channels, int bufferSize);
295 
296     int32_t appPid_ { 0 };
297     std::string appName_;
298     bool speakerAliveStatus_ = true;
299     std::atomic<bool> isInVoIPCall_ = false;
300     std::mutex voipStatusChangeMutex_;
301     std::atomic<int64_t> firstAudioFramePts_{-1};
302     std::atomic<int64_t> firstVideoFramePts_{-1};
303     std::atomic<int64_t> lastWriteAudioFramePts_{0};
304     std::atomic<int64_t> lastMicAudioFramePts_{0};
305     AVScreenCaptureMixBufferType lastWriteType_ = AVScreenCaptureMixBufferType::INVALID;
306     int32_t stableStopInnerSwitchCount_ = 0;
307     bool mixModeAddAudioMicFrame_ = false;
308     bool isInWaitMicSyncState_ = false;
309     AVScreenCaptureMixMode type_;
310     ScreenCaptureServer* screenCaptureServer_;
311 
312     static constexpr int32_t INNER_SWITCH_MIC_REQUIRE_COUNT = 10;
313     static constexpr int32_t ADS_LOG_SKIP_NUM = 1000;
314     static constexpr int64_t MAX_INNER_AUDIO_TIMEOUT_IN_NS = 2000000000; // 2s
315     static constexpr int64_t AUDIO_MIC_TOO_CLOSE_LIMIT_IN_NS = 10000000; // 10ms
316     static constexpr int64_t AUDIO_INTERVAL_IN_NS = 21333334; // 20ms
317     static constexpr int64_t NEG_AUDIO_INTERVAL_IN_NS = -21333334; // 20ms
318     static constexpr int64_t SEC_TO_NS = 1000000000; // 1s
319     static constexpr int64_t MAX_MIC_BEFORE_INNER_TIME_IN_NS = 40000000; // 40ms
320 };
321 
322 class PrivateWindowListenerInScreenCapture : public DisplayManager::IPrivateWindowListener {
323 public:
324     explicit PrivateWindowListenerInScreenCapture(std::weak_ptr<ScreenCaptureServer> screenCaptureServer);
325     ~PrivateWindowListenerInScreenCapture() = default;
326     void OnPrivateWindow(bool hasPrivate) override;
327 
328 private:
329     std::weak_ptr<ScreenCaptureServer> screenCaptureServer_;
330 };
331 
332 class ScreenRendererAudioStateChangeCallback : public AudioRendererStateChangeCallback {
333 public:
334     void OnRendererStateChange(const std::vector<std::shared_ptr<AudioRendererChangeInfo>> &audioRendererChangeInfos);
335     void SetAudioSource(std::shared_ptr<AudioDataSource> audioSource);
336     void SetAppName(std::string appName);
337 private:
338     std::shared_ptr<AudioDataSource> audioSource_ = nullptr;
339     std::string appName_;
340 };
341 
342 class ScreenConnectListenerForSC : public Rosen::ScreenManager::IScreenListener {
343 public:
ScreenConnectListenerForSC(uint64_t screenId,std::weak_ptr<ScreenCaptureServer> screenCaptureServer)344     explicit ScreenConnectListenerForSC(uint64_t screenId, std::weak_ptr<ScreenCaptureServer> screenCaptureServer)
345         : screenId_(screenId), screenCaptureServer_(screenCaptureServer) {}
346     void OnConnect(Rosen::ScreenId screenId);
347     void OnDisconnect(Rosen::ScreenId screenId);
348     void OnChange(Rosen::ScreenId screenId);
349 private:
350     uint64_t screenId_ = SCREEN_ID_INVALID;
351     std::weak_ptr<ScreenCaptureServer> screenCaptureServer_;
352 };
353 
354 class SCWindowLifecycleListener : public Rosen::SessionLifecycleListenerStub {
355 public:
356     explicit SCWindowLifecycleListener(std::weak_ptr<ScreenCaptureServer> screenCaptureServer);
357     ~SCWindowLifecycleListener() override = default;
358     void OnLifecycleEvent(SessionLifecycleEvent event, const LifecycleEventPayload& payload) override;
359 
360 private:
361     std::weak_ptr<ScreenCaptureServer> screenCaptureServer_;
362 };
363 
364 class SCDeathRecipientListener : public IRemoteObject::DeathRecipient {
365 public:
366     using ListenerDiedHandler = std::function<void(const wptr<IRemoteObject>&)>;
SCDeathRecipientListener(ListenerDiedHandler handler)367     explicit SCDeathRecipientListener(ListenerDiedHandler handler) : diedHandler_(std::move(handler)) {}
368     ~SCDeathRecipientListener() override = default;
OnRemoteDied(const wptr<IRemoteObject> & remote)369     void OnRemoteDied(const wptr<IRemoteObject>& remote) final
370     {
371         if (diedHandler_) {
372             diedHandler_(remote);
373         }
374     }
375 
376 private:
377     ListenerDiedHandler diedHandler_;
378 };
379 
380 class SCWindowInfoChangedListener : public Rosen::IWindowInfoChangedListener {
381 public:
382     explicit SCWindowInfoChangedListener(std::weak_ptr<ScreenCaptureServer> screenCaptureServer);
383     ~SCWindowInfoChangedListener() override = default;
384     void OnWindowInfoChanged(const std::vector<std::unordered_map<WindowInfoKey,
385         WindowChangeInfoType>>& windowInfoList) override;
386 
387 private:
388     std::weak_ptr<ScreenCaptureServer> screenCaptureServer_;
389 };
390 
391 class ScreenCaptureSubscriber : public EventFwk::CommonEventSubscriber {
392 public:
ScreenCaptureSubscriber(const EventFwk::CommonEventSubscribeInfo & subscribeInfo,const std::function<void (const EventFwk::CommonEventData &)> & callback)393     ScreenCaptureSubscriber(const EventFwk::CommonEventSubscribeInfo &subscribeInfo,
394         const std::function<void(const EventFwk::CommonEventData &)> &callback)
395         : EventFwk::CommonEventSubscriber(subscribeInfo), callback_(callback)
396     {}
397 
~ScreenCaptureSubscriber()398     ~ScreenCaptureSubscriber()
399     {}
400 
OnReceiveEvent(const EventFwk::CommonEventData & data)401     void OnReceiveEvent(const EventFwk::CommonEventData &data) override
402     {
403         if (callback_ != nullptr) {
404             callback_(data);
405         }
406     }
407 
408 private:
409     std::function<void(const EventFwk::CommonEventData &)> callback_;
410 };
411 } // namespace Media
412 } // namespace OHOS
413 #endif // SCREEN_CAPTURE_SERVICE_SERVER_BASE_H
414