1 /*
2 LZ4 auto-framing library
3 Copyright (C) 2011-2016, Yann Collet.
4
5 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are
9 met:
10
11 * Redistributions of source code must retain the above copyright
12 notice, this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following disclaimer
15 in the documentation and/or other materials provided with the
16 distribution.
17
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 You can contact the author at :
31 - LZ4 homepage : http://www.lz4.org
32 - LZ4 source repository : https://github.com/lz4/lz4
33 */
34
35 /* LZ4F is a stand-alone API to create LZ4-compressed Frames
36 * in full conformance with specification v1.5.0
37 * All related operations, including memory management, are handled by the library.
38 * */
39
40
41 /*-************************************
42 * Compiler Options
43 **************************************/
44 #ifdef _MSC_VER /* Visual Studio */
45 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
46 #endif
47
48
49 /*-************************************
50 * Tuning parameters
51 **************************************/
52 /*
53 * LZ4F_HEAPMODE :
54 * Select how default compression functions will allocate memory for their hash table,
55 * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
56 */
57 #ifndef LZ4F_HEAPMODE
58 # define LZ4F_HEAPMODE 0
59 #endif
60
61
62 /*-************************************
63 * Memory routines
64 **************************************/
65 #include <stdlib.h> /* malloc, calloc, free */
66 #define ALLOC(s) malloc(s)
67 #define ALLOC_AND_ZERO(s) calloc(1,s)
68 #define FREEMEM free
69 #include <string.h> /* memset, memcpy, memmove */
70 #define MEM_INIT memset
71
72
73 /*-************************************
74 * Includes
75 **************************************/
76 #define LZ4F_STATIC_LINKING_ONLY
77 #include "lz4frame.h"
78 #define LZ4_STATIC_LINKING_ONLY
79 #include "lz4.h"
80 #define LZ4_HC_STATIC_LINKING_ONLY
81 #include "lz4hc.h"
82 #define XXH_STATIC_LINKING_ONLY
83 #include "xxhash.h"
84
85
86 /*-************************************
87 * Debug
88 **************************************/
89 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
90 # include <assert.h>
91 #else
92 # ifndef assert
93 # define assert(condition) ((void)0)
94 # endif
95 #endif
96
97 #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
98
99 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)
100 # include <stdio.h>
101 static int g_debuglog_enable = 1;
102 # define DEBUGLOG(l, ...) { \
103 if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
104 fprintf(stderr, __FILE__ ": "); \
105 fprintf(stderr, __VA_ARGS__); \
106 fprintf(stderr, " \n"); \
107 } }
108 #else
109 # define DEBUGLOG(l, ...) {} /* disabled */
110 #endif
111
112
113 /*-************************************
114 * Basic Types
115 **************************************/
116 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
117 # include <stdint.h>
118 typedef uint8_t BYTE;
119 typedef uint16_t U16;
120 typedef uint32_t U32;
121 typedef int32_t S32;
122 typedef uint64_t U64;
123 #else
124 typedef unsigned char BYTE;
125 typedef unsigned short U16;
126 typedef unsigned int U32;
127 typedef signed int S32;
128 typedef unsigned long long U64;
129 #endif
130
131
132 /* unoptimized version; solves endianess & alignment issues */
LZ4F_readLE32(const void * src)133 static U32 LZ4F_readLE32 (const void* src)
134 {
135 const BYTE* const srcPtr = (const BYTE*)src;
136 U32 value32 = srcPtr[0];
137 value32 += (srcPtr[1]<<8);
138 value32 += (srcPtr[2]<<16);
139 value32 += ((U32)srcPtr[3])<<24;
140 return value32;
141 }
142
LZ4F_writeLE32(void * dst,U32 value32)143 static void LZ4F_writeLE32 (void* dst, U32 value32)
144 {
145 BYTE* const dstPtr = (BYTE*)dst;
146 dstPtr[0] = (BYTE)value32;
147 dstPtr[1] = (BYTE)(value32 >> 8);
148 dstPtr[2] = (BYTE)(value32 >> 16);
149 dstPtr[3] = (BYTE)(value32 >> 24);
150 }
151
LZ4F_readLE64(const void * src)152 static U64 LZ4F_readLE64 (const void* src)
153 {
154 const BYTE* const srcPtr = (const BYTE*)src;
155 U64 value64 = srcPtr[0];
156 value64 += ((U64)srcPtr[1]<<8);
157 value64 += ((U64)srcPtr[2]<<16);
158 value64 += ((U64)srcPtr[3]<<24);
159 value64 += ((U64)srcPtr[4]<<32);
160 value64 += ((U64)srcPtr[5]<<40);
161 value64 += ((U64)srcPtr[6]<<48);
162 value64 += ((U64)srcPtr[7]<<56);
163 return value64;
164 }
165
LZ4F_writeLE64(void * dst,U64 value64)166 static void LZ4F_writeLE64 (void* dst, U64 value64)
167 {
168 BYTE* const dstPtr = (BYTE*)dst;
169 dstPtr[0] = (BYTE)value64;
170 dstPtr[1] = (BYTE)(value64 >> 8);
171 dstPtr[2] = (BYTE)(value64 >> 16);
172 dstPtr[3] = (BYTE)(value64 >> 24);
173 dstPtr[4] = (BYTE)(value64 >> 32);
174 dstPtr[5] = (BYTE)(value64 >> 40);
175 dstPtr[6] = (BYTE)(value64 >> 48);
176 dstPtr[7] = (BYTE)(value64 >> 56);
177 }
178
179
180 /*-************************************
181 * Constants
182 **************************************/
183 #define KB *(1<<10)
184 #define MB *(1<<20)
185 #define GB *(1<<30)
186
187 #define _1BIT 0x01
188 #define _2BITS 0x03
189 #define _3BITS 0x07
190 #define _4BITS 0x0F
191 #define _8BITS 0xFF
192
193 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
194 #define LZ4F_MAGICNUMBER 0x184D2204U
195 #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
196 #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
197
198 static const size_t minFHSize = 7;
199 static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
200 static const size_t BHSize = 4;
201
202
203 /*-************************************
204 * Structures and local types
205 **************************************/
206 typedef struct LZ4F_cctx_s
207 {
208 LZ4F_preferences_t prefs;
209 U32 version;
210 U32 cStage;
211 const LZ4F_CDict* cdict;
212 size_t maxBlockSize;
213 size_t maxBufferSize;
214 BYTE* tmpBuff;
215 BYTE* tmpIn;
216 size_t tmpInSize;
217 U64 totalInSize;
218 XXH32_state_t xxh;
219 void* lz4CtxPtr;
220 U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
221 U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
222 } LZ4F_cctx_t;
223
224
225 /*-************************************
226 * Error management
227 **************************************/
228 #define LZ4F_GENERATE_STRING(STRING) #STRING,
229 static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
230
231
LZ4F_isError(LZ4F_errorCode_t code)232 unsigned LZ4F_isError(LZ4F_errorCode_t code)
233 {
234 return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode));
235 }
236
LZ4F_getErrorName(LZ4F_errorCode_t code)237 const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
238 {
239 static const char* codeError = "Unspecified error code";
240 if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)];
241 return codeError;
242 }
243
LZ4F_getErrorCode(size_t functionResult)244 LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
245 {
246 if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError;
247 return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
248 }
249
err0r(LZ4F_errorCodes code)250 static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
251 {
252 /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
253 LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
254 return (LZ4F_errorCode_t)-(ptrdiff_t)code;
255 }
256
LZ4F_getVersion(void)257 unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
258
LZ4F_compressionLevel_max(void)259 int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
260
261
262 /*-************************************
263 * Private functions
264 **************************************/
265 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
266
LZ4F_getBlockSize(unsigned blockSizeID)267 static size_t LZ4F_getBlockSize(unsigned blockSizeID)
268 {
269 static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
270
271 if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
272 blockSizeID -= 4;
273 if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid);
274 return blockSizes[blockSizeID];
275 }
276
LZ4F_headerChecksum(const void * header,size_t length)277 static BYTE LZ4F_headerChecksum (const void* header, size_t length)
278 {
279 U32 const xxh = XXH32(header, length, 0);
280 return (BYTE)(xxh >> 8);
281 }
282
283
284 /*-************************************
285 * Simple-pass compression functions
286 **************************************/
LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,const size_t srcSize)287 static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
288 const size_t srcSize)
289 {
290 LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
291 size_t maxBlockSize = 64 KB;
292 while (requestedBSID > proposedBSID) {
293 if (srcSize <= maxBlockSize)
294 return proposedBSID;
295 proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1);
296 maxBlockSize <<= 2;
297 }
298 return requestedBSID;
299 }
300
301 /*! LZ4F_compressBound_internal() :
302 * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
303 * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
304 * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
305 * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
306 */
LZ4F_compressBound_internal(size_t srcSize,const LZ4F_preferences_t * preferencesPtr,size_t alreadyBuffered)307 static size_t LZ4F_compressBound_internal(size_t srcSize,
308 const LZ4F_preferences_t* preferencesPtr,
309 size_t alreadyBuffered)
310 {
311 LZ4F_preferences_t prefsNull;
312 MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
313 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
314 { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
315 U32 const flush = prefsPtr->autoFlush | (srcSize==0);
316 LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
317 size_t const blockSize = LZ4F_getBlockSize(blockID);
318 size_t const maxBuffered = blockSize - 1;
319 size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
320 size_t const maxSrcSize = srcSize + bufferedSize;
321 unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);
322 size_t const partialBlockSize = maxSrcSize & (blockSize-1);
323 size_t const lastBlockSize = flush ? partialBlockSize : 0;
324 unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
325
326 size_t const blockHeaderSize = 4;
327 size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
328 size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
329
330 return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
331 (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
332 }
333 }
334
LZ4F_compressFrameBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)335 size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
336 {
337 LZ4F_preferences_t prefs;
338 size_t const headerSize = maxFHSize; /* max header size, including optional fields */
339
340 if (preferencesPtr!=NULL) prefs = *preferencesPtr;
341 else MEM_INIT(&prefs, 0, sizeof(prefs));
342 prefs.autoFlush = 1;
343
344 return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
345 }
346
347
348 /*! LZ4F_compressFrame_usingCDict() :
349 * Compress srcBuffer using a dictionary, in a single step.
350 * cdict can be NULL, in which case, no dictionary is used.
351 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
352 * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
353 * however, it's the only way to provide a dictID, so it's not recommended.
354 * @return : number of bytes written into dstBuffer,
355 * or an error code if it fails (can be tested using LZ4F_isError())
356 */
LZ4F_compressFrame_usingCDict(LZ4F_cctx * cctx,void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)357 size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
358 void* dstBuffer, size_t dstCapacity,
359 const void* srcBuffer, size_t srcSize,
360 const LZ4F_CDict* cdict,
361 const LZ4F_preferences_t* preferencesPtr)
362 {
363 LZ4F_preferences_t prefs;
364 LZ4F_compressOptions_t options;
365 BYTE* const dstStart = (BYTE*) dstBuffer;
366 BYTE* dstPtr = dstStart;
367 BYTE* const dstEnd = dstStart + dstCapacity;
368
369 if (preferencesPtr!=NULL)
370 prefs = *preferencesPtr;
371 else
372 MEM_INIT(&prefs, 0, sizeof(prefs));
373 if (prefs.frameInfo.contentSize != 0)
374 prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
375
376 prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
377 prefs.autoFlush = 1;
378 if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
379 prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
380
381 MEM_INIT(&options, 0, sizeof(options));
382 options.stableSrc = 1;
383
384 if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
385 return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
386
387 { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
388 if (LZ4F_isError(headerSize)) return headerSize;
389 dstPtr += headerSize; /* header size */ }
390
391 { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
392 if (LZ4F_isError(cSize)) return cSize;
393 dstPtr += cSize; }
394
395 { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */
396 if (LZ4F_isError(tailSize)) return tailSize;
397 dstPtr += tailSize; }
398
399 return (dstPtr - dstStart);
400 }
401
402
403 /*! LZ4F_compressFrame() :
404 * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
405 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
406 * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
407 * @return : number of bytes written into dstBuffer.
408 * or an error code if it fails (can be tested using LZ4F_isError())
409 */
LZ4F_compressFrame(void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_preferences_t * preferencesPtr)410 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
411 const void* srcBuffer, size_t srcSize,
412 const LZ4F_preferences_t* preferencesPtr)
413 {
414 size_t result;
415 #if (LZ4F_HEAPMODE)
416 LZ4F_cctx_t *cctxPtr;
417 result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
418 if (LZ4F_isError(result)) return result;
419 #else
420 LZ4F_cctx_t cctx;
421 LZ4_stream_t lz4ctx;
422 LZ4F_cctx_t *cctxPtr = &cctx;
423
424 DEBUGLOG(4, "LZ4F_compressFrame");
425 MEM_INIT(&cctx, 0, sizeof(cctx));
426 cctx.version = LZ4F_VERSION;
427 cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
428 if (preferencesPtr == NULL ||
429 preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
430 {
431 LZ4_resetStream(&lz4ctx);
432 cctxPtr->lz4CtxPtr = &lz4ctx;
433 cctxPtr->lz4CtxAlloc = 1;
434 cctxPtr->lz4CtxState = 1;
435 }
436 #endif
437
438 result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
439 srcBuffer, srcSize,
440 NULL, preferencesPtr);
441
442 #if (LZ4F_HEAPMODE)
443 LZ4F_freeCompressionContext(cctxPtr);
444 #else
445 if (preferencesPtr != NULL &&
446 preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
447 {
448 FREEMEM(cctxPtr->lz4CtxPtr);
449 }
450 #endif
451 return result;
452 }
453
454
455 /*-***************************************************
456 * Dictionary compression
457 *****************************************************/
458
459 struct LZ4F_CDict_s {
460 void* dictContent;
461 LZ4_stream_t* fastCtx;
462 LZ4_streamHC_t* HCCtx;
463 }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
464
465 /*! LZ4F_createCDict() :
466 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
467 * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
468 * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
469 * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
470 * @return : digested dictionary for compression, or NULL if failed */
LZ4F_createCDict(const void * dictBuffer,size_t dictSize)471 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
472 {
473 const char* dictStart = (const char*)dictBuffer;
474 LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
475 DEBUGLOG(4, "LZ4F_createCDict");
476 if (!cdict) return NULL;
477 if (dictSize > 64 KB) {
478 dictStart += dictSize - 64 KB;
479 dictSize = 64 KB;
480 }
481 cdict->dictContent = ALLOC(dictSize);
482 cdict->fastCtx = LZ4_createStream();
483 cdict->HCCtx = LZ4_createStreamHC();
484 if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
485 LZ4F_freeCDict(cdict);
486 return NULL;
487 }
488 memcpy(cdict->dictContent, dictStart, dictSize);
489 LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
490 LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
491 LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
492 return cdict;
493 }
494
LZ4F_freeCDict(LZ4F_CDict * cdict)495 void LZ4F_freeCDict(LZ4F_CDict* cdict)
496 {
497 if (cdict==NULL) return; /* support free on NULL */
498 FREEMEM(cdict->dictContent);
499 LZ4_freeStream(cdict->fastCtx);
500 LZ4_freeStreamHC(cdict->HCCtx);
501 FREEMEM(cdict);
502 }
503
504
505 /*-*********************************
506 * Advanced compression functions
507 ***********************************/
508
509 /*! LZ4F_createCompressionContext() :
510 * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
511 * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
512 * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
513 * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
514 * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
515 * Object can release its memory using LZ4F_freeCompressionContext();
516 */
LZ4F_createCompressionContext(LZ4F_compressionContext_t * LZ4F_compressionContextPtr,unsigned version)517 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
518 {
519 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
520 if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
521
522 cctxPtr->version = version;
523 cctxPtr->cStage = 0; /* Next stage : init stream */
524
525 *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
526
527 return LZ4F_OK_NoError;
528 }
529
530
LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)531 LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)
532 {
533 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
534
535 if (cctxPtr != NULL) { /* support free on NULL */
536 FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
537 FREEMEM(cctxPtr->tmpBuff);
538 FREEMEM(LZ4F_compressionContext);
539 }
540
541 return LZ4F_OK_NoError;
542 }
543
544
545 /**
546 * This function prepares the internal LZ4(HC) stream for a new compression,
547 * resetting the context and attaching the dictionary, if there is one.
548 *
549 * It needs to be called at the beginning of each independent compression
550 * stream (i.e., at the beginning of a frame in blockLinked mode, or at the
551 * beginning of each block in blockIndependent mode).
552 */
LZ4F_initStream(void * ctx,const LZ4F_CDict * cdict,int level,LZ4F_blockMode_t blockMode)553 static void LZ4F_initStream(void* ctx,
554 const LZ4F_CDict* cdict,
555 int level,
556 LZ4F_blockMode_t blockMode) {
557 if (level < LZ4HC_CLEVEL_MIN) {
558 if (cdict != NULL || blockMode == LZ4F_blockLinked) {
559 /* In these cases, we will call LZ4_compress_fast_continue(),
560 * which needs an already reset context. Otherwise, we'll call a
561 * one-shot API. The non-continued APIs internally perform their own
562 * resets at the beginning of their calls, where they know what
563 * tableType they need the context to be in. So in that case this
564 * would be misguided / wasted work. */
565 LZ4_resetStream_fast((LZ4_stream_t*)ctx);
566 }
567 LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
568 } else {
569 LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
570 LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
571 }
572 }
573
574
575 /*! LZ4F_compressBegin_usingCDict() :
576 * init streaming compression and writes frame header into dstBuffer.
577 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
578 * @return : number of bytes written into dstBuffer for the header
579 * or an error code (can be tested using LZ4F_isError())
580 */
LZ4F_compressBegin_usingCDict(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)581 size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
582 void* dstBuffer, size_t dstCapacity,
583 const LZ4F_CDict* cdict,
584 const LZ4F_preferences_t* preferencesPtr)
585 {
586 LZ4F_preferences_t prefNull;
587 BYTE* const dstStart = (BYTE*)dstBuffer;
588 BYTE* dstPtr = dstStart;
589 BYTE* headerStart;
590
591 if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
592 MEM_INIT(&prefNull, 0, sizeof(prefNull));
593 if (preferencesPtr == NULL) preferencesPtr = &prefNull;
594 cctxPtr->prefs = *preferencesPtr;
595
596 /* Ctx Management */
597 { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
598 if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
599 FREEMEM(cctxPtr->lz4CtxPtr);
600 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
601 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
602 } else {
603 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
604 }
605 if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
606 cctxPtr->lz4CtxAlloc = ctxTypeID;
607 cctxPtr->lz4CtxState = ctxTypeID;
608 } else if (cctxPtr->lz4CtxState != ctxTypeID) {
609 /* otherwise, a sufficient buffer is allocated, but we need to
610 * reset it to the correct context type */
611 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
612 LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
613 } else {
614 LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
615 }
616 cctxPtr->lz4CtxState = ctxTypeID;
617 }
618 }
619
620 /* Buffer Management */
621 if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
622 cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
623 cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
624
625 { size_t const requiredBuffSize = preferencesPtr->autoFlush ?
626 (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
627 cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
628
629 if (cctxPtr->maxBufferSize < requiredBuffSize) {
630 cctxPtr->maxBufferSize = 0;
631 FREEMEM(cctxPtr->tmpBuff);
632 cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
633 if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
634 cctxPtr->maxBufferSize = requiredBuffSize;
635 } }
636 cctxPtr->tmpIn = cctxPtr->tmpBuff;
637 cctxPtr->tmpInSize = 0;
638 XXH32_reset(&(cctxPtr->xxh), 0);
639
640 /* context init */
641 cctxPtr->cdict = cdict;
642 if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
643 /* frame init only for blockLinked : blockIndependent will be init at each block */
644 LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
645 }
646 if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
647 LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
648 }
649
650 /* Magic Number */
651 LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
652 dstPtr += 4;
653 headerStart = dstPtr;
654
655 /* FLG Byte */
656 *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
657 + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
658 + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
659 + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
660 + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
661 + (cctxPtr->prefs.frameInfo.dictID > 0) );
662 /* BD Byte */
663 *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
664 /* Optional Frame content size field */
665 if (cctxPtr->prefs.frameInfo.contentSize) {
666 LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
667 dstPtr += 8;
668 cctxPtr->totalInSize = 0;
669 }
670 /* Optional dictionary ID field */
671 if (cctxPtr->prefs.frameInfo.dictID) {
672 LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
673 dstPtr += 4;
674 }
675 /* Header CRC Byte */
676 *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
677 dstPtr++;
678
679 cctxPtr->cStage = 1; /* header written, now request input data block */
680 return (dstPtr - dstStart);
681 }
682
683
684 /*! LZ4F_compressBegin() :
685 * init streaming compression and writes frame header into dstBuffer.
686 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
687 * preferencesPtr can be NULL, in which case default parameters are selected.
688 * @return : number of bytes written into dstBuffer for the header
689 * or an error code (can be tested using LZ4F_isError())
690 */
LZ4F_compressBegin(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_preferences_t * preferencesPtr)691 size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
692 void* dstBuffer, size_t dstCapacity,
693 const LZ4F_preferences_t* preferencesPtr)
694 {
695 return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
696 NULL, preferencesPtr);
697 }
698
699
700 /* LZ4F_compressBound() :
701 * @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario.
702 * LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario.
703 * This function cannot fail.
704 */
LZ4F_compressBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)705 size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
706 {
707 return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1);
708 }
709
710
711 typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
712
713
714 /*! LZ4F_makeBlock():
715 * compress a single block, add header and checksum
716 * assumption : dst buffer capacity is >= srcSize */
LZ4F_makeBlock(void * dst,const void * src,size_t srcSize,compressFunc_t compress,void * lz4ctx,int level,const LZ4F_CDict * cdict,LZ4F_blockChecksum_t crcFlag)717 static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
718 compressFunc_t compress, void* lz4ctx, int level,
719 const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
720 {
721 BYTE* const cSizePtr = (BYTE*)dst;
722 U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
723 (int)(srcSize), (int)(srcSize-1),
724 level, cdict);
725 LZ4F_writeLE32(cSizePtr, cSize);
726 if (cSize == 0) { /* compression failed */
727 cSize = (U32)srcSize;
728 LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
729 memcpy(cSizePtr+4, src, srcSize);
730 }
731 if (crcFlag) {
732 U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
733 LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
734 }
735 return 4 + cSize + ((U32)crcFlag)*4;
736 }
737
738
LZ4F_compressBlock(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)739 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
740 {
741 int const acceleration = (level < 0) ? -level + 1 : 1;
742 LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
743 if (cdict) {
744 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
745 } else {
746 return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
747 }
748 }
749
LZ4F_compressBlock_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)750 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
751 {
752 int const acceleration = (level < 0) ? -level + 1 : 1;
753 (void)cdict; /* init once at beginning of frame */
754 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
755 }
756
LZ4F_compressBlockHC(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)757 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
758 {
759 LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
760 if (cdict) {
761 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
762 }
763 return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
764 }
765
LZ4F_compressBlockHC_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)766 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
767 {
768 (void)level; (void)cdict; /* init once at beginning of frame */
769 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
770 }
771
LZ4F_selectCompression(LZ4F_blockMode_t blockMode,int level)772 static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
773 {
774 if (level < LZ4HC_CLEVEL_MIN) {
775 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
776 return LZ4F_compressBlock_continue;
777 }
778 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
779 return LZ4F_compressBlockHC_continue;
780 }
781
LZ4F_localSaveDict(LZ4F_cctx_t * cctxPtr)782 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
783 {
784 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
785 return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
786 return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
787 }
788
789 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
790
791 /*! LZ4F_compressUpdate() :
792 * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
793 * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
794 * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
795 * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
796 * or an error code if it fails (which can be tested using LZ4F_isError())
797 */
LZ4F_compressUpdate(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_compressOptions_t * compressOptionsPtr)798 size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
799 void* dstBuffer, size_t dstCapacity,
800 const void* srcBuffer, size_t srcSize,
801 const LZ4F_compressOptions_t* compressOptionsPtr)
802 {
803 LZ4F_compressOptions_t cOptionsNull;
804 size_t const blockSize = cctxPtr->maxBlockSize;
805 const BYTE* srcPtr = (const BYTE*)srcBuffer;
806 const BYTE* const srcEnd = srcPtr + srcSize;
807 BYTE* const dstStart = (BYTE*)dstBuffer;
808 BYTE* dstPtr = dstStart;
809 LZ4F_lastBlockStatus lastBlockCompressed = notDone;
810 compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
811
812 DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
813
814 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
815 if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
816 return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
817 MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
818 if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
819
820 /* complete tmp buffer */
821 if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
822 size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
823 if (sizeToCopy > srcSize) {
824 /* add src to tmpIn buffer */
825 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
826 srcPtr = srcEnd;
827 cctxPtr->tmpInSize += srcSize;
828 /* still needs some CRC */
829 } else {
830 /* complete tmpIn block and then compress it */
831 lastBlockCompressed = fromTmpBuffer;
832 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
833 srcPtr += sizeToCopy;
834
835 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
836 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
837 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
838
839 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
840 cctxPtr->tmpInSize = 0;
841 }
842 }
843
844 while ((size_t)(srcEnd - srcPtr) >= blockSize) {
845 /* compress full blocks */
846 lastBlockCompressed = fromSrcBuffer;
847 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
848 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
849 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
850 srcPtr += blockSize;
851 }
852
853 if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
854 /* compress remaining input < blockSize */
855 lastBlockCompressed = fromSrcBuffer;
856 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
857 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
858 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
859 srcPtr = srcEnd;
860 }
861
862 /* preserve dictionary if necessary */
863 if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
864 if (compressOptionsPtr->stableSrc) {
865 cctxPtr->tmpIn = cctxPtr->tmpBuff;
866 } else {
867 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
868 if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
869 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
870 }
871 }
872
873 /* keep tmpIn within limits */
874 if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
875 && !(cctxPtr->prefs.autoFlush))
876 {
877 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
878 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
879 }
880
881 /* some input data left, necessarily < blockSize */
882 if (srcPtr < srcEnd) {
883 /* fill tmp buffer */
884 size_t const sizeToCopy = srcEnd - srcPtr;
885 memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
886 cctxPtr->tmpInSize = sizeToCopy;
887 }
888
889 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
890 XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
891
892 cctxPtr->totalInSize += srcSize;
893 return dstPtr - dstStart;
894 }
895
896
897 /*! LZ4F_flush() :
898 * Should you need to create compressed data immediately, without waiting for a block to be filled,
899 * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
900 * The result of the function is the number of bytes written into dstBuffer
901 * (it can be zero, this means there was no data left within compressionContext)
902 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
903 * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
904 */
LZ4F_flush(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_compressOptions_t * compressOptionsPtr)905 size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
906 {
907 BYTE* const dstStart = (BYTE*)dstBuffer;
908 BYTE* dstPtr = dstStart;
909 compressFunc_t compress;
910
911 if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
912 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
913 if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */
914 (void)compressOptionsPtr; /* not yet useful */
915
916 /* select compression function */
917 compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
918
919 /* compress tmp buffer */
920 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
921 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
922 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
923 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
924 cctxPtr->tmpInSize = 0;
925
926 /* keep tmpIn within limits */
927 if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
928 int realDictSize = LZ4F_localSaveDict(cctxPtr);
929 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
930 }
931
932 return dstPtr - dstStart;
933 }
934
935
936 /*! LZ4F_compressEnd() :
937 * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
938 * It will flush whatever data remained within compressionContext (like LZ4_flush())
939 * but also properly finalize the frame, with an endMark and a checksum.
940 * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
941 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
942 * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
943 * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
944 */
LZ4F_compressEnd(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstMaxSize,const LZ4F_compressOptions_t * compressOptionsPtr)945 size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
946 {
947 BYTE* const dstStart = (BYTE*)dstBuffer;
948 BYTE* dstPtr = dstStart;
949
950 size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
951 if (LZ4F_isError(flushSize)) return flushSize;
952 dstPtr += flushSize;
953
954 LZ4F_writeLE32(dstPtr, 0);
955 dstPtr+=4; /* endMark */
956
957 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
958 U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
959 LZ4F_writeLE32(dstPtr, xxh);
960 dstPtr+=4; /* content Checksum */
961 }
962
963 cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */
964 cctxPtr->maxBufferSize = 0; /* reuse HC context */
965
966 if (cctxPtr->prefs.frameInfo.contentSize) {
967 if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
968 return err0r(LZ4F_ERROR_frameSize_wrong);
969 }
970
971 return dstPtr - dstStart;
972 }
973
974
975 /*-***************************************************
976 * Frame Decompression
977 *****************************************************/
978
979 typedef enum {
980 dstage_getFrameHeader=0, dstage_storeFrameHeader,
981 dstage_init,
982 dstage_getBlockHeader, dstage_storeBlockHeader,
983 dstage_copyDirect, dstage_getBlockChecksum,
984 dstage_getCBlock, dstage_storeCBlock,
985 dstage_flushOut,
986 dstage_getSuffix, dstage_storeSuffix,
987 dstage_getSFrameSize, dstage_storeSFrameSize,
988 dstage_skipSkippable
989 } dStage_t;
990
991 struct LZ4F_dctx_s {
992 LZ4F_frameInfo_t frameInfo;
993 U32 version;
994 dStage_t dStage;
995 U64 frameRemainingSize;
996 size_t maxBlockSize;
997 size_t maxBufferSize;
998 BYTE* tmpIn;
999 size_t tmpInSize;
1000 size_t tmpInTarget;
1001 BYTE* tmpOutBuffer;
1002 const BYTE* dict;
1003 size_t dictSize;
1004 BYTE* tmpOut;
1005 size_t tmpOutSize;
1006 size_t tmpOutStart;
1007 XXH32_state_t xxh;
1008 XXH32_state_t blockChecksum;
1009 BYTE header[LZ4F_HEADER_SIZE_MAX];
1010 }; /* typedef'd to LZ4F_dctx in lz4frame.h */
1011
1012
1013 /*! LZ4F_createDecompressionContext() :
1014 * Create a decompressionContext object, which will track all decompression operations.
1015 * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
1016 * Object can later be released using LZ4F_freeDecompressionContext().
1017 * @return : if != 0, there was an error during context creation.
1018 */
LZ4F_createDecompressionContext(LZ4F_dctx ** LZ4F_decompressionContextPtr,unsigned versionNumber)1019 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
1020 {
1021 LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
1022 if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
1023
1024 dctx->version = versionNumber;
1025 *LZ4F_decompressionContextPtr = dctx;
1026 return LZ4F_OK_NoError;
1027 }
1028
LZ4F_freeDecompressionContext(LZ4F_dctx * dctx)1029 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
1030 {
1031 LZ4F_errorCode_t result = LZ4F_OK_NoError;
1032 if (dctx != NULL) { /* can accept NULL input, like free() */
1033 result = (LZ4F_errorCode_t)dctx->dStage;
1034 FREEMEM(dctx->tmpIn);
1035 FREEMEM(dctx->tmpOutBuffer);
1036 FREEMEM(dctx);
1037 }
1038 return result;
1039 }
1040
1041
1042 /*==--- Streaming Decompression operations ---==*/
1043
LZ4F_resetDecompressionContext(LZ4F_dctx * dctx)1044 void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
1045 {
1046 dctx->dStage = dstage_getFrameHeader;
1047 dctx->dict = NULL;
1048 dctx->dictSize = 0;
1049 }
1050
1051
1052 /*! LZ4F_headerSize() :
1053 * @return : size of frame header
1054 * or an error code, which can be tested using LZ4F_isError()
1055 */
LZ4F_headerSize(const void * src,size_t srcSize)1056 static size_t LZ4F_headerSize(const void* src, size_t srcSize)
1057 {
1058 /* minimal srcSize to determine header size */
1059 if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete);
1060
1061 /* special case : skippable frames */
1062 if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
1063
1064 /* control magic number */
1065 if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
1066 return err0r(LZ4F_ERROR_frameType_unknown);
1067
1068 /* Frame Header Size */
1069 { BYTE const FLG = ((const BYTE*)src)[4];
1070 U32 const contentSizeFlag = (FLG>>3) & _1BIT;
1071 U32 const dictIDFlag = FLG & _1BIT;
1072 return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
1073 }
1074 }
1075
1076
1077 /*! LZ4F_decodeHeader() :
1078 * input : `src` points at the **beginning of the frame**
1079 * output : set internal values of dctx, such as
1080 * dctx->frameInfo and dctx->dStage.
1081 * Also allocates internal buffers.
1082 * @return : nb Bytes read from src (necessarily <= srcSize)
1083 * or an error code (testable with LZ4F_isError())
1084 */
LZ4F_decodeHeader(LZ4F_dctx * dctx,const void * src,size_t srcSize)1085 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
1086 {
1087 unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
1088 size_t frameHeaderSize;
1089 const BYTE* srcPtr = (const BYTE*)src;
1090
1091 /* need to decode header to get frameInfo */
1092 if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
1093 MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
1094
1095 /* special case : skippable frames */
1096 if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
1097 dctx->frameInfo.frameType = LZ4F_skippableFrame;
1098 if (src == (void*)(dctx->header)) {
1099 dctx->tmpInSize = srcSize;
1100 dctx->tmpInTarget = 8;
1101 dctx->dStage = dstage_storeSFrameSize;
1102 return srcSize;
1103 } else {
1104 dctx->dStage = dstage_getSFrameSize;
1105 return 4;
1106 }
1107 }
1108
1109 /* control magic number */
1110 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
1111 return err0r(LZ4F_ERROR_frameType_unknown);
1112 dctx->frameInfo.frameType = LZ4F_frame;
1113
1114 /* Flags */
1115 { U32 const FLG = srcPtr[4];
1116 U32 const version = (FLG>>6) & _2BITS;
1117 blockChecksumFlag = (FLG>>4) & _1BIT;
1118 blockMode = (FLG>>5) & _1BIT;
1119 contentSizeFlag = (FLG>>3) & _1BIT;
1120 contentChecksumFlag = (FLG>>2) & _1BIT;
1121 dictIDFlag = FLG & _1BIT;
1122 /* validate */
1123 if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1124 if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
1125 }
1126
1127 /* Frame Header Size */
1128 frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
1129
1130 if (srcSize < frameHeaderSize) {
1131 /* not enough input to fully decode frame header */
1132 if (srcPtr != dctx->header)
1133 memcpy(dctx->header, srcPtr, srcSize);
1134 dctx->tmpInSize = srcSize;
1135 dctx->tmpInTarget = frameHeaderSize;
1136 dctx->dStage = dstage_storeFrameHeader;
1137 return srcSize;
1138 }
1139
1140 { U32 const BD = srcPtr[5];
1141 blockSizeID = (BD>>4) & _3BITS;
1142 /* validate */
1143 if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1144 if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
1145 if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
1146 }
1147
1148 /* check header */
1149 { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
1150 if (HC != srcPtr[frameHeaderSize-1])
1151 return err0r(LZ4F_ERROR_headerChecksum_invalid);
1152 }
1153
1154 /* save */
1155 dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
1156 dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
1157 dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
1158 dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
1159 dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
1160 if (contentSizeFlag)
1161 dctx->frameRemainingSize =
1162 dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
1163 if (dictIDFlag)
1164 dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
1165
1166 dctx->dStage = dstage_init;
1167
1168 return frameHeaderSize;
1169 }
1170
1171
1172 /*! LZ4F_getFrameInfo() :
1173 * This function extracts frame parameters (max blockSize, frame checksum, etc.).
1174 * Usage is optional. Objective is to provide relevant information for allocation purposes.
1175 * This function works in 2 situations :
1176 * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
1177 * Amount of input data provided must be large enough to successfully decode the frame header.
1178 * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
1179 * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
1180 * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
1181 * Decompression must resume from (srcBuffer + *srcSizePtr).
1182 * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
1183 * or an error code which can be tested using LZ4F_isError()
1184 * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
1185 * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
1186 */
LZ4F_getFrameInfo(LZ4F_dctx * dctx,LZ4F_frameInfo_t * frameInfoPtr,const void * srcBuffer,size_t * srcSizePtr)1187 LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
1188 const void* srcBuffer, size_t* srcSizePtr)
1189 {
1190 if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
1191 /* frameInfo already decoded */
1192 size_t o=0, i=0;
1193 *srcSizePtr = 0;
1194 *frameInfoPtr = dctx->frameInfo;
1195 /* returns : recommended nb of bytes for LZ4F_decompress() */
1196 return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
1197 } else {
1198 if (dctx->dStage == dstage_storeFrameHeader) {
1199 /* frame decoding already started, in the middle of header => automatic fail */
1200 *srcSizePtr = 0;
1201 return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
1202 } else {
1203 size_t decodeResult;
1204 size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
1205 if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
1206 if (*srcSizePtr < hSize) {
1207 *srcSizePtr=0;
1208 return err0r(LZ4F_ERROR_frameHeader_incomplete);
1209 }
1210
1211 decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
1212 if (LZ4F_isError(decodeResult)) {
1213 *srcSizePtr = 0;
1214 } else {
1215 *srcSizePtr = decodeResult;
1216 decodeResult = BHSize; /* block header size */
1217 }
1218 *frameInfoPtr = dctx->frameInfo;
1219 return decodeResult;
1220 } }
1221 }
1222
1223
1224 /* LZ4F_updateDict() :
1225 * only used for LZ4F_blockLinked mode */
LZ4F_updateDict(LZ4F_dctx * dctx,const BYTE * dstPtr,size_t dstSize,const BYTE * dstBufferStart,unsigned withinTmp)1226 static void LZ4F_updateDict(LZ4F_dctx* dctx,
1227 const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
1228 unsigned withinTmp)
1229 {
1230 if (dctx->dictSize==0)
1231 dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
1232
1233 if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */
1234 dctx->dictSize += dstSize;
1235 return;
1236 }
1237
1238 if (dstPtr - dstBufferStart + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */
1239 dctx->dict = (const BYTE*)dstBufferStart;
1240 dctx->dictSize = dstPtr - dstBufferStart + dstSize;
1241 return;
1242 }
1243
1244 assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
1245
1246 /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
1247
1248 if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */
1249 /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
1250 assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
1251 dctx->dictSize += dstSize;
1252 return;
1253 }
1254
1255 if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
1256 size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1257 size_t copySize = 64 KB - dctx->tmpOutSize;
1258 const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1259 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1260 if (copySize > preserveSize) copySize = preserveSize;
1261
1262 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1263
1264 dctx->dict = dctx->tmpOutBuffer;
1265 dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
1266 return;
1267 }
1268
1269 if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
1270 if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
1271 size_t const preserveSize = 64 KB - dstSize;
1272 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1273 dctx->dictSize = preserveSize;
1274 }
1275 memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
1276 dctx->dictSize += dstSize;
1277 return;
1278 }
1279
1280 /* join dict & dest into tmp */
1281 { size_t preserveSize = 64 KB - dstSize;
1282 if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
1283 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1284 memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
1285 dctx->dict = dctx->tmpOutBuffer;
1286 dctx->dictSize = preserveSize + dstSize;
1287 }
1288 }
1289
1290
1291
1292 /*! LZ4F_decompress() :
1293 * Call this function repetitively to regenerate compressed data in srcBuffer.
1294 * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
1295 * into dstBuffer of capacity *dstSizePtr.
1296 *
1297 * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
1298 *
1299 * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
1300 * If number of bytes read is < number of bytes provided, then decompression operation is not complete.
1301 * Remaining data will have to be presented again in a subsequent invocation.
1302 *
1303 * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
1304 * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
1305 * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
1306 * Note that this is just a hint, and it's always possible to any srcSize value.
1307 * When a frame is fully decoded, @return will be 0.
1308 * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
1309 */
LZ4F_decompress(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const LZ4F_decompressOptions_t * decompressOptionsPtr)1310 size_t LZ4F_decompress(LZ4F_dctx* dctx,
1311 void* dstBuffer, size_t* dstSizePtr,
1312 const void* srcBuffer, size_t* srcSizePtr,
1313 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1314 {
1315 LZ4F_decompressOptions_t optionsNull;
1316 const BYTE* const srcStart = (const BYTE*)srcBuffer;
1317 const BYTE* const srcEnd = srcStart + *srcSizePtr;
1318 const BYTE* srcPtr = srcStart;
1319 BYTE* const dstStart = (BYTE*)dstBuffer;
1320 BYTE* const dstEnd = dstStart + *dstSizePtr;
1321 BYTE* dstPtr = dstStart;
1322 const BYTE* selectedIn = NULL;
1323 unsigned doAnotherStage = 1;
1324 size_t nextSrcSizeHint = 1;
1325
1326
1327 MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
1328 if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
1329 *srcSizePtr = 0;
1330 *dstSizePtr = 0;
1331
1332 /* behaves as a state machine */
1333
1334 while (doAnotherStage) {
1335
1336 switch(dctx->dStage)
1337 {
1338
1339 case dstage_getFrameHeader:
1340 if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
1341 size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
1342 if (LZ4F_isError(hSize)) return hSize;
1343 srcPtr += hSize;
1344 break;
1345 }
1346 dctx->tmpInSize = 0;
1347 if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
1348 dctx->tmpInTarget = minFHSize; /* minimum size to decode header */
1349 dctx->dStage = dstage_storeFrameHeader;
1350 /* fall-through */
1351
1352 case dstage_storeFrameHeader:
1353 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
1354 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1355 dctx->tmpInSize += sizeToCopy;
1356 srcPtr += sizeToCopy;
1357 }
1358 if (dctx->tmpInSize < dctx->tmpInTarget) {
1359 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
1360 doAnotherStage = 0; /* not enough src data, ask for some more */
1361 break;
1362 }
1363 { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
1364 if (LZ4F_isError(hSize)) return hSize;
1365 }
1366 break;
1367
1368 case dstage_init:
1369 if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
1370 /* internal buffers allocation */
1371 { size_t const bufferNeeded = dctx->maxBlockSize
1372 + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
1373 if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
1374 dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
1375 FREEMEM(dctx->tmpIn);
1376 dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
1377 if (dctx->tmpIn == NULL)
1378 return err0r(LZ4F_ERROR_allocation_failed);
1379 FREEMEM(dctx->tmpOutBuffer);
1380 dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
1381 if (dctx->tmpOutBuffer== NULL)
1382 return err0r(LZ4F_ERROR_allocation_failed);
1383 dctx->maxBufferSize = bufferNeeded;
1384 } }
1385 dctx->tmpInSize = 0;
1386 dctx->tmpInTarget = 0;
1387 dctx->tmpOut = dctx->tmpOutBuffer;
1388 dctx->tmpOutStart = 0;
1389 dctx->tmpOutSize = 0;
1390
1391 dctx->dStage = dstage_getBlockHeader;
1392 /* fall-through */
1393
1394 case dstage_getBlockHeader:
1395 if ((size_t)(srcEnd - srcPtr) >= BHSize) {
1396 selectedIn = srcPtr;
1397 srcPtr += BHSize;
1398 } else {
1399 /* not enough input to read cBlockSize field */
1400 dctx->tmpInSize = 0;
1401 dctx->dStage = dstage_storeBlockHeader;
1402 }
1403
1404 if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */
1405 case dstage_storeBlockHeader:
1406 { size_t const remainingInput = (size_t)(srcEnd - srcPtr);
1407 size_t const wantedData = BHSize - dctx->tmpInSize;
1408 size_t const sizeToCopy = MIN(wantedData, remainingInput);
1409 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1410 srcPtr += sizeToCopy;
1411 dctx->tmpInSize += sizeToCopy;
1412
1413 if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */
1414 nextSrcSizeHint = BHSize - dctx->tmpInSize;
1415 doAnotherStage = 0;
1416 break;
1417 }
1418 selectedIn = dctx->tmpIn;
1419 } /* if (dctx->dStage == dstage_storeBlockHeader) */
1420
1421 /* decode block header */
1422 { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
1423 size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
1424 if (nextCBlockSize==0) { /* frameEnd signal, no more block */
1425 dctx->dStage = dstage_getSuffix;
1426 break;
1427 }
1428 if (nextCBlockSize > dctx->maxBlockSize)
1429 return err0r(LZ4F_ERROR_maxBlockSize_invalid);
1430 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
1431 /* next block is uncompressed */
1432 dctx->tmpInTarget = nextCBlockSize;
1433 if (dctx->frameInfo.blockChecksumFlag) {
1434 XXH32_reset(&dctx->blockChecksum, 0);
1435 }
1436 dctx->dStage = dstage_copyDirect;
1437 break;
1438 }
1439 /* next block is a compressed block */
1440 dctx->tmpInTarget = nextCBlockSize + crcSize;
1441 dctx->dStage = dstage_getCBlock;
1442 if (dstPtr==dstEnd) {
1443 nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
1444 doAnotherStage = 0;
1445 }
1446 break;
1447 }
1448
1449 case dstage_copyDirect: /* uncompressed block */
1450 { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
1451 size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
1452 memcpy(dstPtr, srcPtr, sizeToCopy);
1453 if (dctx->frameInfo.blockChecksumFlag) {
1454 XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
1455 }
1456 if (dctx->frameInfo.contentChecksumFlag)
1457 XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
1458 if (dctx->frameInfo.contentSize)
1459 dctx->frameRemainingSize -= sizeToCopy;
1460
1461 /* history management (linked blocks only)*/
1462 if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
1463 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
1464
1465 srcPtr += sizeToCopy;
1466 dstPtr += sizeToCopy;
1467 if (sizeToCopy == dctx->tmpInTarget) { /* all done */
1468 if (dctx->frameInfo.blockChecksumFlag) {
1469 dctx->tmpInSize = 0;
1470 dctx->dStage = dstage_getBlockChecksum;
1471 } else
1472 dctx->dStage = dstage_getBlockHeader; /* new block */
1473 break;
1474 }
1475 dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
1476 nextSrcSizeHint = dctx->tmpInTarget +
1477 + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */
1478 + BHSize /* next header size */;
1479 doAnotherStage = 0;
1480 break;
1481 }
1482
1483 /* check block checksum for recently transferred uncompressed block */
1484 case dstage_getBlockChecksum:
1485 { const void* crcSrc;
1486 if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
1487 crcSrc = srcPtr;
1488 srcPtr += 4;
1489 } else {
1490 size_t const stillToCopy = 4 - dctx->tmpInSize;
1491 size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
1492 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1493 dctx->tmpInSize += sizeToCopy;
1494 srcPtr += sizeToCopy;
1495 if (dctx->tmpInSize < 4) { /* all input consumed */
1496 doAnotherStage = 0;
1497 break;
1498 }
1499 crcSrc = dctx->header;
1500 }
1501 { U32 const readCRC = LZ4F_readLE32(crcSrc);
1502 U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
1503 if (readCRC != calcCRC)
1504 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1505 } }
1506 dctx->dStage = dstage_getBlockHeader; /* new block */
1507 break;
1508
1509 case dstage_getCBlock:
1510 if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
1511 dctx->tmpInSize = 0;
1512 dctx->dStage = dstage_storeCBlock;
1513 break;
1514 }
1515 /* input large enough to read full block directly */
1516 selectedIn = srcPtr;
1517 srcPtr += dctx->tmpInTarget;
1518
1519 if (0) /* jump over next block */
1520 case dstage_storeCBlock:
1521 { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;
1522 size_t const inputLeft = (size_t)(srcEnd-srcPtr);
1523 size_t const sizeToCopy = MIN(wantedData, inputLeft);
1524 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1525 dctx->tmpInSize += sizeToCopy;
1526 srcPtr += sizeToCopy;
1527 if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
1528 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
1529 doAnotherStage=0;
1530 break;
1531 }
1532 selectedIn = dctx->tmpIn;
1533 }
1534
1535 /* At this stage, input is large enough to decode a block */
1536 if (dctx->frameInfo.blockChecksumFlag) {
1537 dctx->tmpInTarget -= 4;
1538 assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
1539 { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
1540 U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
1541 if (readBlockCrc != calcBlockCrc)
1542 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1543 } }
1544
1545 if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
1546 const char* dict = (const char*)dctx->dict;
1547 size_t dictSize = dctx->dictSize;
1548 int decodedSize;
1549 if (dict && dictSize > 1 GB) {
1550 /* the dictSize param is an int, avoid truncation / sign issues */
1551 dict += dictSize - 64 KB;
1552 dictSize = 64 KB;
1553 }
1554 /* enough capacity in `dst` to decompress directly there */
1555 decodedSize = LZ4_decompress_safe_usingDict(
1556 (const char*)selectedIn, (char*)dstPtr,
1557 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1558 dict, (int)dictSize);
1559 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
1560 if (dctx->frameInfo.contentChecksumFlag)
1561 XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
1562 if (dctx->frameInfo.contentSize)
1563 dctx->frameRemainingSize -= decodedSize;
1564
1565 /* dictionary management */
1566 if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
1567 LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
1568
1569 dstPtr += decodedSize;
1570 dctx->dStage = dstage_getBlockHeader;
1571 break;
1572 }
1573
1574 /* not enough place into dst : decode into tmpOut */
1575 /* ensure enough place for tmpOut */
1576 if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
1577 if (dctx->dict == dctx->tmpOutBuffer) {
1578 if (dctx->dictSize > 128 KB) {
1579 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
1580 dctx->dictSize = 64 KB;
1581 }
1582 dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
1583 } else { /* dict not within tmp */
1584 size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
1585 dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
1586 } }
1587
1588 /* Decode block */
1589 { const char* dict = (const char*)dctx->dict;
1590 size_t dictSize = dctx->dictSize;
1591 int decodedSize;
1592 if (dict && dictSize > 1 GB) {
1593 /* the dictSize param is an int, avoid truncation / sign issues */
1594 dict += dictSize - 64 KB;
1595 dictSize = 64 KB;
1596 }
1597 decodedSize = LZ4_decompress_safe_usingDict(
1598 (const char*)selectedIn, (char*)dctx->tmpOut,
1599 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1600 dict, (int)dictSize);
1601 if (decodedSize < 0) /* decompression failed */
1602 return err0r(LZ4F_ERROR_decompressionFailed);
1603 if (dctx->frameInfo.contentChecksumFlag)
1604 XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
1605 if (dctx->frameInfo.contentSize)
1606 dctx->frameRemainingSize -= decodedSize;
1607 dctx->tmpOutSize = decodedSize;
1608 dctx->tmpOutStart = 0;
1609 dctx->dStage = dstage_flushOut;
1610 }
1611 /* fall-through */
1612
1613 case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
1614 { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
1615 memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
1616
1617 /* dictionary management */
1618 if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
1619 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);
1620
1621 dctx->tmpOutStart += sizeToCopy;
1622 dstPtr += sizeToCopy;
1623
1624 if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
1625 dctx->dStage = dstage_getBlockHeader; /* get next block */
1626 break;
1627 }
1628 /* could not flush everything : stop there, just request a block header */
1629 doAnotherStage = 0;
1630 nextSrcSizeHint = BHSize;
1631 break;
1632 }
1633
1634 case dstage_getSuffix:
1635 if (dctx->frameRemainingSize)
1636 return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
1637 if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
1638 nextSrcSizeHint = 0;
1639 LZ4F_resetDecompressionContext(dctx);
1640 doAnotherStage = 0;
1641 break;
1642 }
1643 if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */
1644 dctx->tmpInSize = 0;
1645 dctx->dStage = dstage_storeSuffix;
1646 } else {
1647 selectedIn = srcPtr;
1648 srcPtr += 4;
1649 }
1650
1651 if (dctx->dStage == dstage_storeSuffix) /* can be skipped */
1652 case dstage_storeSuffix:
1653 { size_t const remainingInput = (size_t)(srcEnd - srcPtr);
1654 size_t const wantedData = 4 - dctx->tmpInSize;
1655 size_t const sizeToCopy = MIN(wantedData, remainingInput);
1656 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1657 srcPtr += sizeToCopy;
1658 dctx->tmpInSize += sizeToCopy;
1659 if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
1660 nextSrcSizeHint = 4 - dctx->tmpInSize;
1661 doAnotherStage=0;
1662 break;
1663 }
1664 selectedIn = dctx->tmpIn;
1665 } /* if (dctx->dStage == dstage_storeSuffix) */
1666
1667 /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
1668 { U32 const readCRC = LZ4F_readLE32(selectedIn);
1669 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
1670 if (readCRC != resultCRC)
1671 return err0r(LZ4F_ERROR_contentChecksum_invalid);
1672 nextSrcSizeHint = 0;
1673 LZ4F_resetDecompressionContext(dctx);
1674 doAnotherStage = 0;
1675 break;
1676 }
1677
1678 case dstage_getSFrameSize:
1679 if ((srcEnd - srcPtr) >= 4) {
1680 selectedIn = srcPtr;
1681 srcPtr += 4;
1682 } else {
1683 /* not enough input to read cBlockSize field */
1684 dctx->tmpInSize = 4;
1685 dctx->tmpInTarget = 8;
1686 dctx->dStage = dstage_storeSFrameSize;
1687 }
1688
1689 if (dctx->dStage == dstage_storeSFrameSize)
1690 case dstage_storeSFrameSize:
1691 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
1692 (size_t)(srcEnd - srcPtr) );
1693 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1694 srcPtr += sizeToCopy;
1695 dctx->tmpInSize += sizeToCopy;
1696 if (dctx->tmpInSize < dctx->tmpInTarget) {
1697 /* not enough input to get full sBlockSize; wait for more */
1698 nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
1699 doAnotherStage = 0;
1700 break;
1701 }
1702 selectedIn = dctx->header + 4;
1703 } /* if (dctx->dStage == dstage_storeSFrameSize) */
1704
1705 /* case dstage_decodeSFrameSize: */ /* no direct entry */
1706 { size_t const SFrameSize = LZ4F_readLE32(selectedIn);
1707 dctx->frameInfo.contentSize = SFrameSize;
1708 dctx->tmpInTarget = SFrameSize;
1709 dctx->dStage = dstage_skipSkippable;
1710 break;
1711 }
1712
1713 case dstage_skipSkippable:
1714 { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
1715 srcPtr += skipSize;
1716 dctx->tmpInTarget -= skipSize;
1717 doAnotherStage = 0;
1718 nextSrcSizeHint = dctx->tmpInTarget;
1719 if (nextSrcSizeHint) break; /* still more to skip */
1720 /* frame fully skipped : prepare context for a new frame */
1721 LZ4F_resetDecompressionContext(dctx);
1722 break;
1723 }
1724 } /* switch (dctx->dStage) */
1725 } /* while (doAnotherStage) */
1726
1727 /* preserve history within tmp whenever necessary */
1728 LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
1729 if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
1730 && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
1731 && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */
1732 && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */
1733 {
1734 if (dctx->dStage == dstage_flushOut) {
1735 size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1736 size_t copySize = 64 KB - dctx->tmpOutSize;
1737 const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1738 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1739 if (copySize > preserveSize) copySize = preserveSize;
1740
1741 if (copySize > 0)
1742 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1743
1744 dctx->dict = dctx->tmpOutBuffer;
1745 dctx->dictSize = preserveSize + dctx->tmpOutStart;
1746 } else {
1747 const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;
1748 size_t const newDictSize = MIN(dctx->dictSize, 64 KB);
1749
1750 if (newDictSize > 0)
1751 memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
1752
1753 dctx->dict = dctx->tmpOutBuffer;
1754 dctx->dictSize = newDictSize;
1755 dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
1756 }
1757 }
1758
1759 *srcSizePtr = (srcPtr - srcStart);
1760 *dstSizePtr = (dstPtr - dstStart);
1761 return nextSrcSizeHint;
1762 }
1763
1764 /*! LZ4F_decompress_usingDict() :
1765 * Same as LZ4F_decompress(), using a predefined dictionary.
1766 * Dictionary is used "in place", without any preprocessing.
1767 * It must remain accessible throughout the entire frame decoding.
1768 */
LZ4F_decompress_usingDict(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const void * dict,size_t dictSize,const LZ4F_decompressOptions_t * decompressOptionsPtr)1769 size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
1770 void* dstBuffer, size_t* dstSizePtr,
1771 const void* srcBuffer, size_t* srcSizePtr,
1772 const void* dict, size_t dictSize,
1773 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1774 {
1775 if (dctx->dStage <= dstage_init) {
1776 dctx->dict = (const BYTE*)dict;
1777 dctx->dictSize = dictSize;
1778 }
1779 return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
1780 srcBuffer, srcSizePtr,
1781 decompressOptionsPtr);
1782 }
1783