1 /*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "pkg_algo_lz4.h"
16 #include "lz4.h"
17 #include "lz4frame.h"
18 #include "lz4hc.h"
19 #include "pkg_stream.h"
20 #include "pkg_utils.h"
21 #include "securec.h"
22
23 namespace hpackage {
PkgAlgorithmLz4(const Lz4FileInfo & config)24 PkgAlgorithmLz4::PkgAlgorithmLz4(const Lz4FileInfo &config) : PkgAlgorithm(),
25 compressionLevel_(config.compressionLevel),
26 blockIndependence_(config.blockIndependence),
27 contentChecksumFlag_(config.contentChecksumFlag),
28 blockSizeID_(config.blockSizeID),
29 autoFlush_(config.autoFlush)
30 {
31 // blockIndependence_ 0 LZ4F_blockLinked
32 // contentChecksumFlag_ 0 disable
33 // blockSizeID_ LZ4F_default=0
34 if (compressionLevel_ < 1) {
35 compressionLevel_ = 2;
36 }
37 if (compressionLevel_ >= LZ4HC_CLEVEL_MAX) {
38 compressionLevel_ = LZ4HC_CLEVEL_MAX;
39 }
40 }
41
AdpLz4Compress(const uint8_t * src,uint8_t * dest,uint32_t srcSize,uint32_t dstCapacity) const42 int32_t PkgAlgorithmLz4::AdpLz4Compress(const uint8_t *src, uint8_t *dest,
43 uint32_t srcSize, uint32_t dstCapacity) const
44 {
45 if (compressionLevel_ < LZ4HC_CLEVEL_MIN) { // hc 最小是3
46 return LZ4_compress_default(reinterpret_cast<const char *>(src), reinterpret_cast<char *>(dest),
47 (int32_t)srcSize, (int32_t)dstCapacity);
48 }
49 return LZ4_compress_HC(reinterpret_cast<const char *>(src), reinterpret_cast<char *>(dest), srcSize, dstCapacity,
50 compressionLevel_);
51 }
52
AdpLz4Decompress(const uint8_t * src,uint8_t * dest,uint32_t srcSize,uint32_t dstCapacity) const53 int32_t PkgAlgorithmLz4::AdpLz4Decompress(const uint8_t *src, uint8_t *dest, uint32_t srcSize,
54 uint32_t dstCapacity) const
55 {
56 return LZ4_decompress_safe(reinterpret_cast<const char *>(src), reinterpret_cast<char *>(dest), srcSize,
57 dstCapacity);
58 }
59
Pack(const PkgStreamPtr inStream,const PkgStreamPtr outStream,PkgAlgorithmContext & context)60 int32_t PkgAlgorithmBlockLz4::Pack(const PkgStreamPtr inStream, const PkgStreamPtr outStream,
61 PkgAlgorithmContext &context)
62 {
63 PKG_CHECK(inStream != nullptr && outStream != nullptr, return PKG_INVALID_PARAM, "Param context null!");
64 size_t blockSize = GetBlockSizeFromBlockId(blockSizeID_);
65 blockSize = (blockSize > LZ4B_BLOCK_SIZE) ? LZ4B_BLOCK_SIZE : blockSize;
66 PkgBuffer inBuffer = {blockSize};
67 PkgBuffer outBuffer = {LZ4_compressBound(blockSize)};
68 PKG_CHECK(inBuffer.buffer != nullptr && outBuffer.buffer != nullptr,
69 return PKG_NONE_MEMORY, "Fail to alloc buffer ");
70
71 PKG_LOGI("frameInfo blockSizeID %d compressionLevel_: %d blockIndependence_:%d contentChecksumFlag_:%d %zu",
72 static_cast<int32_t>(blockSizeID_), static_cast<int32_t>(compressionLevel_),
73 static_cast<int32_t>(blockIndependence_), static_cast<int32_t>(contentChecksumFlag_), blockSize);
74
75 size_t srcOffset = context.srcOffset;
76 size_t destOffset = context.destOffset;
77 size_t remainSize = context.unpackedSize;
78 size_t readLen = 0;
79 /* 写包头 */
80 WriteLE32(outBuffer.buffer, LZ4B_MAGIC_NUMBER);
81 int32_t ret = outStream->Write(outBuffer, sizeof(LZ4B_MAGIC_NUMBER), destOffset);
82 PKG_CHECK(ret == PKG_SUCCESS, return ret, "Fail write data ");
83 destOffset += sizeof(LZ4B_MAGIC_NUMBER);
84
85 while (remainSize > 0) {
86 ret = ReadData(inStream, srcOffset, inBuffer, remainSize, readLen);
87 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail read data ");
88
89 // Compress Block, reserve 4 bytes to store block size
90 int32_t outSize = AdpLz4Compress(inBuffer.buffer,
91 outBuffer.buffer + LZ4B_REVERSED_LEN, readLen, outBuffer.length - LZ4B_REVERSED_LEN);
92 PKG_CHECK(outSize > 0, break, "Fail to compress data outSize %d ", outSize);
93
94 // Write block to buffer.
95 // Buffer format: <block size> + <block contents>
96 WriteLE32(outBuffer.buffer, outSize);
97 ret = outStream->Write(outBuffer, outSize + LZ4B_REVERSED_LEN, destOffset);
98 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail write data ");
99
100 srcOffset += readLen;
101 destOffset += outSize + LZ4B_REVERSED_LEN;
102 }
103 PKG_CHECK(srcOffset - context.srcOffset == context.unpackedSize,
104 return ret, "original size error %zu %zu", srcOffset, context.unpackedSize);
105 context.packedSize = destOffset - context.destOffset;
106
107 return PKG_SUCCESS;
108 }
109
Unpack(const PkgStreamPtr inStream,const PkgStreamPtr outStream,PkgAlgorithmContext & context)110 int32_t PkgAlgorithmBlockLz4::Unpack(const PkgStreamPtr inStream, const PkgStreamPtr outStream,
111 PkgAlgorithmContext &context)
112 {
113 PKG_CHECK(inStream != nullptr && outStream != nullptr, return PKG_INVALID_PARAM, "Param context null!");
114 size_t inBuffSize = LZ4_compressBound(LZ4B_BLOCK_SIZE);
115 PKG_CHECK(inBuffSize > 0, return PKG_NONE_MEMORY, "BufferSize must > 0");
116 PkgBuffer inBuffer(inBuffSize);
117 PkgBuffer outBuffer(LZ4B_BLOCK_SIZE);
118 PKG_CHECK(inBuffer.buffer != nullptr && outBuffer.buffer != nullptr, return PKG_NONE_MEMORY,
119 "Fail to alloc buffer ");
120
121 size_t srcOffset = context.srcOffset;
122 size_t destOffset = context.destOffset;
123 size_t remainSize = context.packedSize;
124 size_t readLen = 0;
125
126 /* Main Loop */
127 while (1) {
128 /* Block Size */
129 inBuffer.length = sizeof(uint32_t);
130 int32_t ret = ReadData(inStream, srcOffset, inBuffer, remainSize, readLen);
131 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail read data ");
132 if (readLen == 0) {
133 break;
134 }
135 uint32_t blockSize = ReadLE32(inBuffer.buffer);
136 PKG_CHECK(!(blockSize > LZ4_COMPRESSBOUND(LZ4B_BLOCK_SIZE) || blockSize > inBuffSize), break,
137 "Fail to get block size %u %u", blockSize, LZ4_COMPRESSBOUND(LZ4B_BLOCK_SIZE));
138 srcOffset += sizeof(uint32_t);
139
140 /* Read Block */
141 inBuffer.length = blockSize;
142 ret = ReadData(inStream, srcOffset, inBuffer, remainSize, readLen);
143 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail read data ");
144
145 /* Decode Block */
146 int32_t decodeSize = AdpLz4Decompress(inBuffer.buffer,
147 outBuffer.buffer, readLen, LZ4B_BLOCK_SIZE);
148 PKG_CHECK(decodeSize > 0, break, "Fail to decompress");
149
150 /* Write Block */
151 ret = outStream->Write(outBuffer, decodeSize, destOffset);
152 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail write data ");
153 destOffset += decodeSize;
154 srcOffset += readLen;
155 }
156 context.packedSize = srcOffset - context.srcOffset;
157 context.unpackedSize = destOffset - context.destOffset;
158 return PKG_SUCCESS;
159 }
160
GetPackParam(LZ4F_compressionContext_t & ctx,LZ4F_preferences_t & preferences,size_t & inBuffSize,size_t & outBuffSize) const161 int32_t PkgAlgorithmLz4::GetPackParam(LZ4F_compressionContext_t &ctx, LZ4F_preferences_t &preferences,
162 size_t &inBuffSize, size_t &outBuffSize) const
163 {
164 LZ4F_errorCode_t errorCode = 0;
165 errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
166 PKG_CHECK(!LZ4F_isError(errorCode), return PKG_NONE_MEMORY,
167 "Fail to create compress context %s", LZ4F_getErrorName(errorCode));
168 size_t blockSize = GetBlockSizeFromBlockId(blockSizeID_);
169 PKG_CHECK(!memset_s(&preferences, sizeof(preferences), 0, sizeof(preferences)),
170 return PKG_NONE_MEMORY, "Memset failed");
171 preferences.autoFlush = autoFlush_;
172 preferences.compressionLevel = compressionLevel_;
173 preferences.frameInfo.blockMode = ((blockIndependence_ == 0) ? LZ4F_blockLinked : LZ4F_blockIndependent);
174 preferences.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID_;
175 preferences.frameInfo.contentChecksumFlag =
176 ((contentChecksumFlag_ == 0) ? LZ4F_noContentChecksum : LZ4F_contentChecksumEnabled);
177
178 outBuffSize = LZ4F_compressBound(blockSize, &preferences);
179 PKG_CHECK(outBuffSize > 0, return PKG_NONE_MEMORY, "BufferSize must > 0");
180 inBuffSize = blockSize;
181
182 PKG_LOGI("frameInfo blockSizeID %d compressionLevel_: %d blockIndependence_:%d contentChecksumFlag_:%d",
183 static_cast<int32_t>(blockSizeID_), static_cast<int32_t>(compressionLevel_),
184 static_cast<int32_t>(blockIndependence_), static_cast<int32_t>(contentChecksumFlag_));
185 PKG_LOGI("blockSize %zu %zu %zu", blockSize, GetBlockSizeFromBlockId(blockSizeID_), outBuffSize);
186 return PKG_SUCCESS;
187 }
188
189 /* 打包数据时，会自动生成magic字 */
Pack(const PkgStreamPtr inStream,const PkgStreamPtr outStream,PkgAlgorithmContext & context)190 int32_t PkgAlgorithmLz4::Pack(const PkgStreamPtr inStream, const PkgStreamPtr outStream,
191 PkgAlgorithmContext &context)
192 {
193 PKG_CHECK(inStream != nullptr && outStream != nullptr, return PKG_INVALID_PARAM, "Param context null!");
194 LZ4F_compressionContext_t ctx;
195 LZ4F_preferences_t preferences;
196 size_t inLength = 0;
197 size_t outLength = 0;
198 int32_t ret = GetPackParam(ctx, preferences, inLength, outLength);
199 PKG_CHECK(ret == PKG_SUCCESS, return PKG_NONE_MEMORY, "Fail to get param for pack");
200
201 PkgBuffer inBuffer(inLength);
202 PkgBuffer outBuffer(outLength);
203 PKG_CHECK(inBuffer.buffer != nullptr && outBuffer.buffer != nullptr,
204 (void)LZ4F_freeCompressionContext(ctx); return PKG_NONE_MEMORY, "Fail to alloc buffer ");
205 size_t srcOffset = context.srcOffset;
206 size_t destOffset = context.destOffset;
207 size_t remainSize = context.unpackedSize;
208
209 /* 写包头 */
210 size_t dataLen = LZ4F_compressBegin(ctx, outBuffer.buffer, outBuffer.length, &preferences);
211 PKG_CHECK(!LZ4F_isError(dataLen), (void)LZ4F_freeCompressionContext(ctx);
212 return PKG_NONE_MEMORY, "Fail to generate header %s", LZ4F_getErrorName(dataLen));
213 ret = outStream->Write(outBuffer, dataLen, destOffset);
214 PKG_CHECK(ret == PKG_SUCCESS, (void)LZ4F_freeCompressionContext(ctx); return ret, "Fail write data ");
215
216 destOffset += dataLen;
217 while (remainSize > 0) {
218 ret = ReadData(inStream, srcOffset, inBuffer, remainSize, dataLen);
219 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail read data ");
220
221 size_t outSize = LZ4F_compressUpdate(ctx,
222 outBuffer.buffer, outBuffer.length, inBuffer.buffer, dataLen, nullptr);
223 PKG_CHECK(!LZ4F_isError(outSize), ret = PKG_NONE_MEMORY; break,
224 "Fail to compress update %s", LZ4F_getErrorName(outSize));
225 ret = outStream->Write(outBuffer, outSize, destOffset);
226 PKG_CHECK(ret == PKG_SUCCESS, ret = PKG_NONE_MEMORY; break, "Fail write data ");
227
228 srcOffset += dataLen;
229 destOffset += outSize;
230 }
231
232 if (ret == PKG_SUCCESS) {
233 size_t headerSize = LZ4F_compressEnd(ctx, outBuffer.buffer, outBuffer.length, nullptr);
234 PKG_CHECK(!LZ4F_isError(headerSize), (void)LZ4F_freeCompressionContext(ctx);
235 return PKG_NONE_MEMORY, "Fail to compress update end %s", LZ4F_getErrorName(headerSize));
236 ret = outStream->Write(outBuffer, headerSize, destOffset);
237 PKG_CHECK(ret == PKG_SUCCESS, (void)LZ4F_freeCompressionContext(ctx); return ret, "Fail write data ");
238 destOffset += headerSize;
239 }
240
241 (void)LZ4F_freeCompressionContext(ctx);
242 PKG_CHECK(srcOffset - context.srcOffset == context.unpackedSize,
243 return ret, "original size error %zu %zu", srcOffset, context.unpackedSize);
244 context.packedSize = destOffset - context.destOffset;
245 return PKG_SUCCESS;
246 }
247
GetUnpackParam(LZ4F_decompressionContext_t & ctx,const PkgStreamPtr inStream,size_t & nextToRead,size_t & srcOffset)248 int32_t PkgAlgorithmLz4::GetUnpackParam(LZ4F_decompressionContext_t &ctx, const PkgStreamPtr inStream,
249 size_t &nextToRead, size_t &srcOffset)
250 {
251 LZ4F_errorCode_t errorCode = 0;
252 LZ4F_frameInfo_t frameInfo;
253
254 errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
255 PKG_CHECK(!LZ4F_isError(errorCode), return PKG_INVALID_LZ4,
256 "Fail to create compress context %s", LZ4F_getErrorName(errorCode));
257
258 PkgBuffer pkgHeader(LZ4S_HEADER_LEN);
259 WriteLE32(pkgHeader.buffer, LZ4S_MAGIC_NUMBER);
260
261 /* Decode stream descriptor */
262 size_t readLen = 0;
263 size_t outBuffSize = 0;
264 size_t inBuffSize = 0;
265 size_t sizeCheck = sizeof(LZ4B_MAGIC_NUMBER);
266 nextToRead = LZ4F_decompress(ctx, nullptr, &outBuffSize, pkgHeader.buffer, &sizeCheck, nullptr);
267 PKG_CHECK(!LZ4F_isError(nextToRead), (void)LZ4F_freeDecompressionContext(ctx);
268 return PKG_INVALID_LZ4, "Fail to decode frame info %s", LZ4F_getErrorName(nextToRead));
269 PKG_CHECK(nextToRead <= pkgHeader.length, (void)LZ4F_freeDecompressionContext(ctx);
270 return PKG_INVALID_LZ4, "Invalid pkgHeader.length %d", pkgHeader.length);
271
272 size_t remainSize = LZ4S_HEADER_LEN;
273 pkgHeader.length = nextToRead;
274 int32_t ret = ReadData(inStream, srcOffset, pkgHeader, remainSize, readLen);
275 PKG_CHECK(ret == PKG_SUCCESS, (void)LZ4F_freeDecompressionContext(ctx); return PKG_INVALID_LZ4, "Fail read data ");
276 PKG_CHECK(readLen == pkgHeader.length, (void)LZ4F_freeDecompressionContext(ctx); return PKG_INVALID_LZ4,
277 "Invalid len %zu %zu", readLen, pkgHeader.length);
278 srcOffset += readLen;
279 sizeCheck = readLen;
280 nextToRead = LZ4F_decompress(ctx, nullptr, &outBuffSize, pkgHeader.buffer, &sizeCheck, nullptr);
281 errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, nullptr, &inBuffSize);
282 PKG_CHECK(!LZ4F_isError(errorCode), (void)LZ4F_freeDecompressionContext(ctx); return PKG_INVALID_LZ4,
283 "Fail to decode frame info %s", LZ4F_getErrorName(errorCode));
284 PKG_CHECK(frameInfo.blockSizeID >= 3 && frameInfo.blockSizeID <= 7,
285 (void)LZ4F_freeDecompressionContext(ctx); return PKG_INVALID_LZ4, "Invalid block size ID %d",
286 frameInfo.blockSizeID);
287
288 blockIndependence_ = frameInfo.blockMode;
289 contentChecksumFlag_ = frameInfo.contentChecksumFlag;
290 blockSizeID_ = frameInfo.blockSizeID;
291 return PKG_SUCCESS;
292 }
293
Unpack(const PkgStreamPtr inStream,const PkgStreamPtr outStream,PkgAlgorithmContext & context)294 int32_t PkgAlgorithmLz4::Unpack(const PkgStreamPtr inStream, const PkgStreamPtr outStream,
295 PkgAlgorithmContext &context)
296 {
297 PKG_CHECK(inStream != nullptr && outStream != nullptr, return PKG_INVALID_PARAM, "Param context null!");
298 LZ4F_decompressionContext_t ctx;
299 LZ4F_errorCode_t errorCode = 0;
300 size_t srcOffset = context.srcOffset;
301 size_t destOffset = context.destOffset;
302 size_t remainSize = context.packedSize;
303 size_t nextToRead = 0;
304 int32_t ret = GetUnpackParam(ctx, inStream, nextToRead, srcOffset);
305 PKG_CHECK(ret == PKG_SUCCESS, return PKG_INVALID_LZ4, "Fail to get param ");
306
307 size_t outBuffSize = GetBlockSizeFromBlockId(blockSizeID_);
308 PKG_LOGI("Block size ID %d outBuffSize:%zu", blockSizeID_, outBuffSize);
309 size_t inBuffSize = outBuffSize + sizeof(uint32_t);
310 PKG_CHECK(inBuffSize > 0 && outBuffSize > 0, return PKG_NONE_MEMORY, "Buffer size must > 0");
311
312 PkgBuffer inBuffer(inBuffSize);
313 PkgBuffer outBuffer(outBuffSize);
314 PKG_CHECK(inBuffer.buffer != nullptr && outBuffer.buffer != nullptr,
315 (void)LZ4F_freeDecompressionContext(ctx); return PKG_NONE_MEMORY, "Fail to alloc buffer ");
316
317 /* Main Loop */
318 while (nextToRead != 0) {
319 size_t readLen = 0;
320 size_t decodedBytes = outBuffSize;
321 PKG_CHECK(nextToRead <= inBuffSize, break, "Error next read %zu %zu ", nextToRead, inBuffSize);
322
323 /* Read Block */
324 inBuffer.length = nextToRead;
325 ret = ReadData(inStream, srcOffset, inBuffer, remainSize, readLen);
326 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail read data ");
327
328 /* Decode Block */
329 size_t sizeCheck = readLen;
330 errorCode = LZ4F_decompress(ctx, outBuffer.buffer,
331 &decodedBytes, inBuffer.buffer, &sizeCheck, nullptr);
332 PKG_CHECK(!LZ4F_isError(errorCode), (void)LZ4F_freeDecompressionContext(ctx); return PKG_INVALID_LZ4,
333 "Fail to decompress %s", LZ4F_getErrorName(errorCode));
334 if (decodedBytes == 0) {
335 srcOffset += readLen;
336 break;
337 }
338 PKG_CHECK(sizeCheck == nextToRead, break, "Error next read %zu %zu ", nextToRead, sizeCheck);
339
340 /* Write Block */
341 ret = outStream->Write(outBuffer, decodedBytes, destOffset);
342 PKG_CHECK(ret == PKG_SUCCESS, break, "Fail write data ");
343 destOffset += decodedBytes;
344 srcOffset += readLen;
345 nextToRead = errorCode;
346 }
347 errorCode = LZ4F_freeDecompressionContext(ctx);
348 PKG_CHECK(!LZ4F_isError(errorCode), return PKG_NONE_MEMORY,
349 "Fail to free compress context %s", LZ4F_getErrorName(errorCode));
350 context.packedSize = srcOffset - context.srcOffset;
351 context.unpackedSize = destOffset - context.destOffset;
352 return ret;
353 }
354
UpdateFileInfo(PkgManager::FileInfoPtr info) const355 void PkgAlgorithmLz4::UpdateFileInfo(PkgManager::FileInfoPtr info) const
356 {
357 Lz4FileInfo *lz4Info = (Lz4FileInfo *)info;
358 lz4Info->fileInfo.packMethod = PKG_COMPRESS_METHOD_LZ4;
359 lz4Info->fileInfo.digestMethod = PKG_DIGEST_TYPE_NONE;
360 lz4Info->compressionLevel = compressionLevel_;
361 lz4Info->blockIndependence = blockIndependence_;
362 lz4Info->contentChecksumFlag = contentChecksumFlag_;
363 lz4Info->blockSizeID = blockSizeID_;
364 lz4Info->autoFlush = autoFlush_;
365 }
366 } // namespace hpackage
367