1 /*
2 * Copyright (C) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef NSTACKX_FILE_MANAGER_H
17 #define NSTACKX_FILE_MANAGER_H
18
19 #include "nstackx_epoll.h"
20 #ifdef MBEDTLS_INCLUDED
21 #include "nstackx_mbedtls.h"
22 #else
23 #include "nstackx_openssl.h"
24 #endif
25 #include "nstackx_dfile_config.h"
26 #include "nstackx_util.h"
27
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31
32 #define NSTACKX_FILE_MANAGER_THREAD_NUM 3
33 #define NSTACKX_MAX_DATA_FWRITE_TIMEOUT_COUNT 30
34 #define MAX_SEND_FILE_OPENED_PER_LIST 10 /* at least 1 */
35
36 #define FILE_MANAGER_EOK 0 /* OK */
37 #define FILE_MANAGER_EMUTEX (-1) /* mutex lock or unlock error */
38 #define FILE_MANAGER_ENOMEM (-2) /* Out of memory */
39 #define FILE_MANAGER_MANAGER_BLIST_EADDFULL (-3) /* try to add node to a full block frame list */
40 #define FILE_MANAGER_LIST_EBLOCK (-3) /* receive a illegal block */
41 #define FILE_MANAGER_FILE_ENOSPC (-3) /* device has no available storage space */
42 #define FILE_MANAGER_FILE_ENOMEM (-4) /* Insufficient kernel memory was available */
43 #define FILE_MANAGER_FILE_ENFILE (-5) /* the number of open file descriptors has been reached */
44 #define FILE_MANAGER_FILE_EACCES (-6) /* permission denied */
45 #define FILE_MANAGER_FILE_ENAMETOOLONG (-7) /* file pathname is too long */
46 #define FILE_MANAGER_FILE_ETXTBSY (-8) /* file is occupied */
47 #define FILE_MANAGER_FILE_EOTHER (-9) /* other error */
48 #define NSTACKX_MEGA_BYTES 1048576
49 #define MEGA_BYTES_TRANSFER_NOTICE_THRESHOLD 20
50 #define NSTACKX_KILO_BYTES 1024
51 #define KILO_BYTES_TRANSFER_NOTICE_THRESHOLD 20480
52 #define FILE_RECV_LIST_LEAST_SIZE 10
53 #define FILE_RECV_LIST_IO_WRITE_THRESHOLD 0.8
54 #define FILE_RECV_LIST_SLOW_START_RATE 2
55
56 #ifdef NSTACKX_WITH_LITEOS
57 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (64 * 1024 * 1024)
58 #else
59 #define FILE_RECV_LIST_MEM_THRESHOLD_WARNING (800 * 1024 * 1024)
60 #endif
61
62 #ifdef BUILD_FOR_WINDOWS
63 #define NSTACKX_INVALID_FD NULL
64 #else
65 #define NSTACKX_INVALID_FD (-1)
66 #endif
67
68 typedef enum {
69 FILE_MANAGER_INNER_ERROR = 1,
70 FILE_MANAGER_SEND_FAIL,
71 FILE_MANAGER_SEND_WAITING_END,
72 FILE_MANAGER_RECEIVE_FAIL,
73 FILE_MANAGER_RECEIVE_SUCCESS,
74 FILE_MANAGER_IN_PROGRESS,
75 FILE_MANAGER_TRANS_IN_PROGRESS
76 } FileManagerMsgType;
77
78 typedef struct {
79 uint64_t fileSize;
80 uint64_t startOffset;
81 uint16_t fileId;
82 char *fileName;
83 } FileBaseInfo;
84
85 typedef struct {
86 FileBaseInfo *fileBasicInfo;
87 uint16_t fileNum;
88 uint16_t transId;
89 uint16_t pathType;
90 uint8_t noSyncFlag;
91 } RecvFileListInfo;
92
93 typedef struct {
94 char *fileList[NSTACKX_DFILE_MAX_FILE_NUM + 1];
95 uint64_t fileSize[NSTACKX_DFILE_MAX_FILE_NUM];
96 uint64_t startOffset[NSTACKX_DFILE_MAX_FILE_NUM];
97 uint16_t fileNum;
98 uint16_t transId;
99 uint8_t tarFlag;
100 uint8_t smallFlag;
101 } SendFileListInfo;
102
103 typedef struct {
104 uint16_t fileId;
105 uint64_t fileSize;
106 uint16_t standardBlockSize;
107 uint32_t totalBlockNum;
108 char *fileName;
109 #ifdef BUILD_FOR_WINDOWS
110 FILE *fd;
111 #else
112 int32_t fd;
113 #endif
114 uint8_t *tarData;
115 int32_t errCode;
116 int64_t maxSequenceSend;
117 uint32_t receivedBlockNum;
118 uint64_t fileOffset;
119 uint64_t writeOffset;
120 uint8_t isEndBlockReceived;
121 uint64_t startOffset;
122 } FileInfo;
123
124 typedef struct {
125 List list;
126 uint16_t fileId;
127 uint32_t blockSequence;
128 uint32_t linkSequence;
129 } SendRetranRequestNode;
130
131 typedef struct {
132 List head;
133 uint32_t maxSize;
134 uint32_t size;
135 pthread_mutex_t lock;
136 } MutexList;
137
138 typedef struct {
139 List list;
140 FileDataFrame *fileDataFrame;
141 uint32_t sendLen;
142 uint8_t socketIndex;
143 } BlockFrame;
144
145 /* Reuse DFileMsg for ease use */
146 typedef DFileMsg FileManagerMsg;
147 typedef void (*FileListMsgReceiver)(uint16_t fileId, FileManagerMsgType msgType, FileManagerMsg *msg, void *context,
148 uint16_t transId);
149
150 typedef struct {
151 FileListMsgReceiver msgReceiver;
152 void *context;
153 } FileListMsgPara;
154
155 typedef enum {
156 FILE_LIST_TRANSFER_FINISH = 1,
157 FILE_LIST_TRANSFER_CANCEL,
158 } TaskStopType;
159
160 typedef struct {
161 uint16_t fileId;
162 uint32_t blockSequence;
163 pthread_mutex_t lock;
164 } SendFilesOutSet;
165
166 typedef struct {
167 List list;
168 uint16_t transId;
169 uint16_t fileNum;
170 int32_t errCode;
171 FileInfo fileInfo[NSTACKX_DFILE_MAX_FILE_NUM];
172 sem_t semStop;
173 TaskStopType stopType;
174 uint32_t runStatus;
175 uint32_t innerRecvSize;
176 uint16_t sendFileProcessed;
177 uint16_t recvFileProcessed;
178 SendFilesOutSet newReadOutSet;
179 MutexList sendRetranList; /* DATA:SendRetranBlock */
180 MutexList recvBlockList; /* DATA:BlockFrame */
181 List innerRecvBlockHead; /* DATA:BlockFrame */
182 EpollDesc epollfd;
183 List *eventNodeChain;
184 FileListMsgReceiver msgReceiver;
185 void *context;
186 CryptPara cryptPara;
187 uint64_t bytesTransferredLastRecord;
188 uint64_t totalBytes;
189 const char *storagePath; /* only useful for receiver */
190 const char *tarFile;
191 FILE *tarFd;
192 FileDataFrame *tarFrame;
193 FileInfo tarFileInfo;
194 uint8_t isOccupied;
195 uint8_t isRecvEmptyFilesCreated;
196 uint8_t socketIndex;
197 uint8_t allFileDataReceived;
198 uint8_t hasUnInsetFrame;
199 uint8_t tarFlag;
200 uint8_t smallFlag;
201 uint8_t noSyncFlag;
202 uint8_t tarFinished;
203 uint16_t blockOffset;
204 uint16_t maxFrameLength;
205 uint32_t bindedSendBlockListIdx;
206 uint32_t dataWriteTimeoutCnt;
207 uint64_t bytesTransferred; /* only useful for non-tar sender */
208 } FileListTask;
209
210 typedef void (*FileManagerMsgReceiver)(FileManagerMsgType msgType, int32_t errCode, void *context);
211
212 typedef struct {
213 char *storagePath;
214 uint16_t pathType;
215 } TypedStoragePath;
216
217 typedef struct {
218 MutexList sendBlockFrameList; /* DATA:BlockFrame */
219 List *sendRetranListTail;
220 sem_t semBlockListNotFull;
221 uint32_t bandingTransNum;
222 } SendBlockFrameListPara;
223
224 typedef struct {
225 uint32_t runStatus;
226 int32_t errCode;
227 uint8_t isSender;
228 uint8_t transFlag;
229 uint8_t recvListOverIo;
230 uint16_t maxFrameLength;
231 uint16_t typedPathNum;
232 sem_t semTaskListNotEmpty;
233 char *commonStoragePath;
234 TypedStoragePath pathList[NSTACKX_MAX_STORAGE_PATH_NUM];
235 MutexList taskList; /* DATA:FileListTask */
236 pthread_t fileManagerTid[NSTACKX_FILE_MANAGER_THREAD_NUM];
237 EpollDesc epollfd;
238 List *eventNodeChain;
239 FileManagerMsgReceiver msgReceiver;
240 void *context;
241 uint8_t key[AES_256_KEY_LENGTH];
242 uint32_t keyLen;
243 uint64_t totalBytes;
244 atomic_t totalPackInMemory;
245 uint64_t stoppedTasksBytesTransferred;
246 uint64_t bytesTransferredLastRecord;
247 atomic_t bytesTransferredInCurPeriod;
248 uint16_t sendFrameListNum;
249 SendBlockFrameListPara sendBlockFrameListPara[NSTACKX_MAX_CLIENT_SEND_THREAD_NUM];
250 uint32_t maxSendBlockListSize;
251 uint32_t maxRecvBlockListSize;
252 uint64_t iorBytes;
253 uint64_t iowBytes;
254 uint32_t iorRate;
255 uint32_t iowRate;
256 uint32_t iowMaxRate;
257 uint32_t sendListFullTimes;
258 uint64_t iowCount; /* io write count in NSTACKX_WLAN_MAX_CONTROL_FRAME_TIMEOUT second */
259 } FileManager;
260
261 typedef struct {
262 EpollDesc epollfd;
263 List *eventNodeChain;
264 FileManagerMsgReceiver msgReceiver;
265 void *context;
266 } FileManagerMsgPara;
267
268 typedef enum {
269 STATE_RECEIVE_ONGOING = 0,
270 STATE_RECEIVE_DONE_FAIL,
271 STATE_RECEIVE_DONE_SUCCESSFULLY,
272 } FileRecvState;
273
274 typedef struct {
275 List list;
276 TransIdState transIdState;
277 uint16_t transId;
278 } TransStateNode;
279
280 #define FILE_MANAGE_RUN 0
281 #define FILE_MANAGE_DESTROY 1
282
283 #define FILE_LIST_STATUS_IDLE 0
284 #define FILE_LIST_STATUS_RUN 1
285 #define FILE_LIST_STATUS_STOP 2
286
287 #define NSTACKX_MAX_RETRAN_BLOCK_NUM 50000
288 #define NSTACKX_MAX_PROCESSING_TASK_NUM 100
289
290 #define LIST_TAIL_BLOCK_REPEAT_TIMES 2
291 #define THREAD_QUIT_TRY_TIMES 3
292 #define GCM_AAD_CHAR 'A'
293
294 #define FILE_MANAGER_THREADS_BINDED_CPU_NUM NSTACKX_FILE_MANAGER_THREAD_NUM
295
CheckManager(const FileManager * fileManager)296 static inline int32_t CheckManager(const FileManager *fileManager)
297 {
298 if (fileManager == NULL || fileManager->runStatus != FILE_MANAGE_RUN || fileManager->errCode != FILE_MANAGER_EOK) {
299 return NSTACKX_EINVAL;
300 }
301 return NSTACKX_EOK;
302 }
303
CheckSenderManager(const FileManager * fileManager)304 static inline int32_t CheckSenderManager(const FileManager *fileManager)
305 {
306 if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_TRUE) {
307 return NSTACKX_EINVAL;
308 }
309 return NSTACKX_EOK;
310 }
311
CheckReceiverManager(FileManager * fileManager)312 static inline int32_t CheckReceiverManager(FileManager *fileManager)
313 {
314 if (CheckManager(fileManager) != NSTACKX_EOK || fileManager->isSender != NSTACKX_FALSE) {
315 return NSTACKX_EINVAL;
316 }
317 return NSTACKX_EOK;
318 }
319
CheckFilelist(FileListTask * fileList)320 static inline int32_t CheckFilelist(FileListTask *fileList)
321 {
322 if (fileList == NULL || fileList->runStatus != FILE_LIST_STATUS_RUN || fileList->errCode != FILE_MANAGER_EOK) {
323 return NSTACKX_EFAILED;
324 }
325 return NSTACKX_EOK;
326 }
327
CheckFilelistNotStop(FileListTask * fileList)328 static inline int32_t CheckFilelistNotStop(FileListTask *fileList)
329 {
330 if (fileList == NULL || fileList->runStatus == FILE_LIST_STATUS_STOP || fileList->errCode != FILE_MANAGER_EOK) {
331 return NSTACKX_EFAILED;
332 }
333 return NSTACKX_EOK;
334 }
335
336 /* Not thread safe */
337 FileManager *FileManagerCreate(uint8_t isSender, FileManagerMsgPara *msgPara, const uint8_t *key,
338 uint32_t keyLen, uint16_t connType);
339
340 /* Not thread safe */
341 void StopFileManagerThreads(FileManager *fileManager);
342
343 /*
344 * Destroy fileManager and free related resource.
345 * Note: this is not thread safe, and you must call the interface StopFileManagerThreads to stop all related threads
346 * before call this interface.
347 */
348 void FileManagerDestroy(FileManager *fileManager);
349
350 /* Not thread safe */
351 int32_t FileManagerSetMaxFrameLength(FileManager *fileManager, uint16_t maxFrameLength);
352
353 /* Not thread safe */
354 int32_t FileManagerSetRecvParaWithConnType(FileManager *fileManager, uint16_t connectType);
355
356 /* Not thread safe */
357 int32_t FileManagerSetWritePath(FileManager *fileManager, const char *storagePath);
358
359 /* Not thread safe */
360 int32_t FileManagerSetWritePathList(FileManager *fileManager, char *path[], uint16_t *pathType, uint16_t pathNum);
361
362 /* Not thread safe */
363 int32_t FileManagerSendFileTask(FileManager *fileManager, const SendFileListInfo *fileListInfo,
364 const FileListMsgPara *msgPara);
365 /* Not thread safe */
366 int32_t FileManagerResetSendOutSet(FileManager *fileManager, uint16_t fileId, uint32_t blockSequence, uint16_t transId);
367
368 /* Not thread safe */
369 int32_t FileManagerFileRead(FileManager *fileManager, uint32_t tid, BlockFrame **block, int32_t nr);
370
371 /* Not thread safe */
372 int32_t FileManagerRecvFileTask(FileManager *fileManager, RecvFileListInfo *fileListInfo, FileListMsgPara *msgPara);
373
374 /* Not thread safe */
375 int32_t FileManagerFileWrite(FileManager *fileManager, FileDataFrame *frame);
376
377 /*
378 * Stop the transfer of file list specialized by the argument transId and clear related resource.
379 * Para: stopType - this is only meaningful for receiver and is one of the follow integer:
380 * FILE_LIST_TRANSFER_FINISH, the receiver will remove all error and incompelte files received in this tranfer;
381 * FILE_LIST_TRANSFER_CANCEL, the receicer will remove all files in this transfer.
382 * Note: this interface is not thread safe
383 */
384 int32_t FileManagerStopTask(FileManager *fileManager, uint16_t transId, TaskStopType stopType);
385
386 /* Not thread safe */
387 uint8_t FileManagerIsLastBlockRead(FileManager *fileManager, uint16_t transId);
388
389 /* Not thread safe */
390 uint8_t FileManagerHasPendingData(FileManager *fileManager);
391
392 /* Not thread safe */
393 int32_t FileManagerGetLastSequence(FileManager *fileManager, uint16_t transId, uint16_t fileId, uint32_t *sequence);
394
395 /* Not thread safe */
396 uint8_t FileManagerIsRecvBlockWritable(FileManager *fileManager, uint16_t transId);
397
398 /* Not thread safe */
399 int32_t FileManagerGetTotalBytes(FileManager *fileManager, uint64_t *totalBytes);
400
401 /* Not thread safe */
402 int32_t FileManagerGetBytesTransferred(FileManager *fileManager, uint64_t *bytesTransferred);
403
404 /* Not thread safe */
405 int32_t FileManagerGetTransUpdateInfo(FileManager *fileManager, uint16_t transId, uint64_t *totalBytes,
406 uint64_t *bytesTransferred);
407
408 int32_t GetEncryptedDataTarFrame(CryptPara *cryptPara, uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
409
410 int32_t GetNoEncryptedDataTarFrame(uint16_t fileId, FileListTask *fileList, uint16_t targetLenth);
411
412 int32_t FileManagerGetReceivedFiles(FileManager *fileManager, uint16_t transId, uint16_t fileIdList[],
413 uint8_t fileIdSuccessFlag[], uint32_t *fileNum);
414
415 int32_t FileManagerSetAllDataReceived(FileManager *fileManager, uint16_t transId);
416
417 uint8_t PushSendBlockFrame(FileManager *fileManager, const FileListTask *fileList, const FileDataFrame *fileDataFrame);
418
419 void UpdateTarFileListSendStatus(FileListTask *fileList);
420
421 FileDataFrame *CreateRetranBlockFrame(FileManager *fileManager, FileListTask *fileList);
422
423 uint8_t PushRetranBlockFrame(FileManager *fileManager, const FileListTask *fileList,
424 const FileDataFrame *fileDataFrame);
425 void FileManagerCLearReadOutSet(FileListTask *fileList);
426
427 uint8_t GetBlockHeadFlag(uint8_t isStartFrame, uint8_t isEndFrame);
428
429 int32_t MutexListInit(MutexList *mutexList, uint32_t maxSize);
430
431 void MutexListDestory(MutexList *mutexList);
432
433 int32_t MutexListPopFront(MutexList *mutexList, List **curFront, uint8_t *isPoped);
434
435 int32_t MutexListAddNode(MutexList *mutexList, List *element, uint8_t isFront);
436
437 void NotifyFileManagerMsg(const FileManager *fileManager, FileManagerMsgType msgType);
438
439 void NotifyFileListMsg(const FileListTask *fileList, FileManagerMsgType msgType);
440
441 void NotifyFileMsg(const FileListTask *fileList, uint16_t fileId, FileManagerMsgType msgType);
442
443 int32_t ConvertErrCode(int32_t error);
444
445 int32_t SetFileOffset(FileInfo *fileInfo, uint64_t fileOffset);
446
447 void CloseFile(FileInfo *fileInfo);
448
449 uint64_t FileListGetBytesTransferred(const FileListTask *fileList, uint8_t isSender);
450
451 uint16_t GetStandardBlockSize(const FileManager *fileManager);
452
453 int32_t SetCryptPara(FileListTask *fileList, const uint8_t key[], uint32_t keyLen);
454
455 FileListTask *GetFileListById(MutexList *taskList, uint16_t transId, uint8_t *isErrorOccurred);
456
457 void RefreshBytesTransFerred(FileManager *fileManager, BlockFrame *frame);
458 int32_t GetFileBlockListSize(MutexList *taskList, uint32_t *recvListAllSize, uint32_t *recvInnerAllSize);
459 extern int32_t SetTransIdState(DFileSession *session, uint16_t transId, TransIdState state);
460 extern TransStateNode *GetTransIdState(DFileSession *session, uint16_t transId, TransIdState *state);
461 extern int32_t IsTransIdDone(DFileSession *session, uint16_t transId);
462 extern void ClearTransStateList(DFileSession *session);
463
464 void FileSync(const FileInfo *fileInfo);
465 char *GetStoragePathByType(FileManager *fileManager, uint16_t pathType);
466 char *GetFullFilePath(const char *path, const char *fileName);
467
468 #ifdef __cplusplus
469 }
470 #endif
471
472 #endif /* NSTACKX_FILE_MANAGER_H */
473